581 lines
23 KiB
Python
Executable File
581 lines
23 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Raid Guild DAO Scraper
|
|
|
|
This script fetches all members of the Raid Guild DAO and stores their
|
|
Ethereum addresses in the database. It also attempts to resolve ENS names
|
|
for the addresses.
|
|
|
|
Raid Guild is a Moloch DAO on Gnosis Chain (formerly xDai).
|
|
|
|
Usage:
|
|
python raid_guild_scraper.py
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import time
|
|
from datetime import datetime
|
|
from typing import Dict, List, Optional, Any
|
|
import requests
|
|
from web3 import Web3
|
|
from dotenv import load_dotenv
|
|
|
|
# Add parent directory to path to import utils
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from utils.db_connector import DatabaseConnector
|
|
from utils.ens_resolver import ENSResolver
|
|
from utils.logger import setup_logger
|
|
|
|
# Load environment variables
|
|
load_dotenv()
|
|
|
|
# Setup logging
|
|
logger = setup_logger("raid_guild_scraper")
|
|
|
|
class RaidGuildScraper:
|
|
"""Scraper for Raid Guild DAO members."""
|
|
|
|
def __init__(self):
|
|
"""Initialize the Raid Guild scraper."""
|
|
self.dao_address = "0xfe1084bc16427e5eb7f13fc19bcd4e641f7d571f"
|
|
self.dao_name = "Raid Guild"
|
|
self.alchemy_api_key = os.getenv("ALCHEMY_API_KEY")
|
|
self.graph_api_key = os.getenv("GRAPH_API_KEY")
|
|
|
|
# Check if we have a Graph API key
|
|
if not self.graph_api_key:
|
|
logger.warning("GRAPH_API_KEY not found in environment variables, using direct subgraph URL")
|
|
# Fallback to direct subgraph URL (may not work)
|
|
self.graph_url = "https://api.thegraph.com/subgraphs/id/2d3CDkKyxhpLDZRLWHMCvWp9cCYdWp4Y7g5ecaBmeqad"
|
|
else:
|
|
# Use the gateway URL with API key
|
|
self.graph_url = f"https://gateway.thegraph.com/api/{self.graph_api_key}/subgraphs/id/2d3CDkKyxhpLDZRLWHMCvWp9cCYdWp4Y7g5ecaBmeqad"
|
|
logger.info("Using The Graph gateway with API key")
|
|
|
|
# Set up Web3 provider for Ethereum mainnet (for ENS resolution)
|
|
provider_url = f"https://eth-mainnet.g.alchemy.com/v2/{self.alchemy_api_key}"
|
|
self.web3 = Web3(Web3.HTTPProvider(provider_url))
|
|
self.db = DatabaseConnector()
|
|
self.ens_resolver = ENSResolver(self.web3)
|
|
|
|
# Validate API keys
|
|
if not self.alchemy_api_key:
|
|
logger.error("ALCHEMY_API_KEY not found in environment variables")
|
|
sys.exit(1)
|
|
|
|
# Register data source
|
|
self.register_data_source()
|
|
|
|
def register_data_source(self) -> None:
|
|
"""Register this DAO as a data source in the database."""
|
|
self.db.upsert_data_source(
|
|
name=f"DAO:{self.dao_name}",
|
|
source_type="DAO",
|
|
description=f"Members of {self.dao_name} DAO ({self.dao_address}) on Gnosis Chain"
|
|
)
|
|
|
|
def get_dao_members(self) -> List[Dict[str, Any]]:
|
|
"""
|
|
Fetch all members of the Raid Guild DAO using The Graph API.
|
|
|
|
Returns:
|
|
List of dictionaries containing member addresses and shares/loot
|
|
"""
|
|
logger.info(f"Fetching members for {self.dao_name} ({self.dao_address})")
|
|
|
|
# Start a scraping job
|
|
job_id = self.db.create_scraping_job(
|
|
source_name=f"DAO:{self.dao_name}",
|
|
status="running"
|
|
)
|
|
|
|
members = []
|
|
try:
|
|
# First, try to get the DAO information to confirm it exists
|
|
query = """
|
|
query GetDao($daoAddress: String!) {
|
|
moloches(where: {id: $daoAddress}) {
|
|
id
|
|
title
|
|
version
|
|
totalShares
|
|
totalLoot
|
|
memberCount
|
|
}
|
|
}
|
|
"""
|
|
|
|
variables = {
|
|
"daoAddress": self.dao_address.lower()
|
|
}
|
|
|
|
# Try the Graph API
|
|
response = requests.post(
|
|
self.graph_url,
|
|
json={"query": query, "variables": variables}
|
|
)
|
|
|
|
if response.status_code != 200:
|
|
logger.error(f"Failed to fetch DAO info: {response.text}")
|
|
self.db.update_scraping_job(job_id, "failed", error_message=f"API error: {response.text}")
|
|
return self.get_hardcoded_members()
|
|
|
|
data = response.json()
|
|
|
|
# Check for errors in the GraphQL response
|
|
if "errors" in data:
|
|
error_message = str(data["errors"])
|
|
logger.error(f"GraphQL error: {error_message}")
|
|
return self.try_direct_contract_query(job_id)
|
|
|
|
# Check if we found the DAO
|
|
dao_data = data.get("data", {}).get("moloches", [])
|
|
if not dao_data:
|
|
logger.warning("DAO not found in The Graph, trying direct contract query")
|
|
return self.try_direct_contract_query(job_id)
|
|
|
|
dao = dao_data[0]
|
|
logger.info(f"Found DAO: {dao.get('title', 'Unknown')} with {dao.get('memberCount', 0)} members")
|
|
|
|
# Now fetch all members
|
|
query = """
|
|
query GetMembers($daoAddress: String!) {
|
|
members(where: {molochAddress: $daoAddress, exists: true}, first: 1000) {
|
|
id
|
|
memberAddress
|
|
createdAt
|
|
shares
|
|
loot
|
|
}
|
|
}
|
|
"""
|
|
|
|
variables = {
|
|
"daoAddress": self.dao_address.lower()
|
|
}
|
|
|
|
response = requests.post(
|
|
self.graph_url,
|
|
json={"query": query, "variables": variables}
|
|
)
|
|
|
|
if response.status_code != 200:
|
|
logger.error(f"Failed to fetch members: {response.text}")
|
|
self.db.update_scraping_job(job_id, "failed", error_message=f"API error: {response.text}")
|
|
return self.get_hardcoded_members()
|
|
|
|
data = response.json()
|
|
|
|
# Check for errors in the GraphQL response
|
|
if "errors" in data:
|
|
error_message = str(data["errors"])
|
|
logger.error(f"GraphQL error when fetching members: {error_message}")
|
|
return self.try_direct_contract_query(job_id)
|
|
|
|
# Process members from the API
|
|
members_data = data.get("data", {}).get("members", [])
|
|
|
|
if not members_data:
|
|
logger.warning("No members found in API response, trying direct contract query")
|
|
return self.try_direct_contract_query(job_id)
|
|
|
|
logger.info(f"Found {len(members_data)} members in API response")
|
|
|
|
# Process members
|
|
for member in members_data:
|
|
address = member.get("memberAddress")
|
|
if not address:
|
|
continue
|
|
|
|
# Get shares and loot
|
|
shares = member.get("shares", "0")
|
|
loot = member.get("loot", "0")
|
|
|
|
# Get join date if available
|
|
joined_at = None
|
|
if "createdAt" in member:
|
|
try:
|
|
joined_at = datetime.fromtimestamp(int(member["createdAt"])).isoformat()
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
members.append({
|
|
"address": address,
|
|
"shares": shares,
|
|
"loot": loot,
|
|
"joined_at": joined_at
|
|
})
|
|
|
|
# Update job with success
|
|
self.db.update_scraping_job(
|
|
job_id=job_id,
|
|
status="completed",
|
|
records_processed=len(members_data),
|
|
records_added=len(members)
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching DAO members: {str(e)}")
|
|
self.db.update_scraping_job(job_id, "failed", error_message=str(e))
|
|
|
|
# Try direct contract query
|
|
logger.info("Trying direct contract query due to error")
|
|
return self.try_direct_contract_query(job_id)
|
|
|
|
logger.info(f"Found {len(members)} DAO members")
|
|
return members
|
|
|
|
def try_direct_contract_query(self, job_id) -> List[Dict[str, Any]]:
|
|
"""
|
|
Try to query the Moloch DAO contract directly using Web3.
|
|
|
|
Args:
|
|
job_id: The ID of the scraping job
|
|
|
|
Returns:
|
|
List of dictionaries containing member addresses
|
|
"""
|
|
logger.info("Attempting to query Moloch DAO contract directly")
|
|
|
|
try:
|
|
# Set up Web3 provider for Gnosis Chain
|
|
gnosis_rpc_url = "https://rpc.gnosischain.com"
|
|
gnosis_web3 = Web3(Web3.HTTPProvider(gnosis_rpc_url))
|
|
|
|
if not gnosis_web3.is_connected():
|
|
logger.error("Failed to connect to Gnosis Chain RPC")
|
|
return self.get_hardcoded_members()
|
|
|
|
# Moloch DAO ABI (minimal for member queries)
|
|
moloch_abi = [
|
|
{
|
|
"constant": True,
|
|
"inputs": [],
|
|
"name": "getMemberCount",
|
|
"outputs": [{"name": "", "type": "uint256"}],
|
|
"payable": False,
|
|
"stateMutability": "view",
|
|
"type": "function"
|
|
},
|
|
{
|
|
"constant": True,
|
|
"inputs": [{"name": "index", "type": "uint256"}],
|
|
"name": "getMemberAddressByIndex",
|
|
"outputs": [{"name": "", "type": "address"}],
|
|
"payable": False,
|
|
"stateMutability": "view",
|
|
"type": "function"
|
|
},
|
|
{
|
|
"constant": True,
|
|
"inputs": [{"name": "memberAddress", "type": "address"}],
|
|
"name": "members",
|
|
"outputs": [
|
|
{"name": "delegateKey", "type": "address"},
|
|
{"name": "shares", "type": "uint256"},
|
|
{"name": "loot", "type": "uint256"},
|
|
{"name": "exists", "type": "bool"},
|
|
{"name": "highestIndexYesVote", "type": "uint256"},
|
|
{"name": "jailed", "type": "uint256"}
|
|
],
|
|
"payable": False,
|
|
"stateMutability": "view",
|
|
"type": "function"
|
|
}
|
|
]
|
|
|
|
# Create contract instance
|
|
contract_address = Web3.to_checksum_address(self.dao_address)
|
|
contract = gnosis_web3.eth.contract(address=contract_address, abi=moloch_abi)
|
|
|
|
# Get member count
|
|
try:
|
|
member_count = contract.functions.getMemberCount().call()
|
|
logger.info(f"Found {member_count} members in the contract")
|
|
except Exception as e:
|
|
logger.error(f"Error getting member count: {str(e)}")
|
|
# Try alternative approach - fetch from DAOhaus UI
|
|
return self.scrape_daohaus_ui(job_id)
|
|
|
|
members = []
|
|
# Fetch each member
|
|
for i in range(member_count):
|
|
try:
|
|
# Get member address
|
|
member_address = contract.functions.getMemberAddressByIndex(i).call()
|
|
|
|
# Get member details
|
|
member_details = contract.functions.members(member_address).call()
|
|
|
|
# Check if member exists
|
|
if member_details[3]: # exists flag
|
|
shares = str(member_details[1])
|
|
loot = str(member_details[2])
|
|
|
|
members.append({
|
|
"address": member_address,
|
|
"shares": shares,
|
|
"loot": loot,
|
|
"joined_at": None # We don't have this information from the contract
|
|
})
|
|
except Exception as e:
|
|
logger.warning(f"Error fetching member at index {i}: {str(e)}")
|
|
continue
|
|
|
|
if members:
|
|
# Update job with success
|
|
self.db.update_scraping_job(
|
|
job_id=job_id,
|
|
status="completed",
|
|
records_processed=member_count,
|
|
records_added=len(members)
|
|
)
|
|
|
|
logger.info(f"Successfully fetched {len(members)} members from the contract")
|
|
return members
|
|
else:
|
|
logger.warning("Failed to fetch members from contract, trying DAOhaus UI scraping")
|
|
return self.scrape_daohaus_ui(job_id)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in direct contract query: {str(e)}")
|
|
return self.scrape_daohaus_ui(job_id)
|
|
|
|
def scrape_daohaus_ui(self, job_id) -> List[Dict[str, Any]]:
|
|
"""
|
|
Attempt to scrape member data from the DAOhaus UI.
|
|
|
|
Args:
|
|
job_id: The ID of the scraping job
|
|
|
|
Returns:
|
|
List of dictionaries containing member addresses
|
|
"""
|
|
logger.info("Attempting to scrape member data from DAOhaus UI")
|
|
|
|
try:
|
|
# DAOhaus API endpoint for members
|
|
url = f"https://api.daohaus.club/dao/0x64/{self.dao_address.lower()}/members"
|
|
|
|
response = requests.get(url)
|
|
|
|
if response.status_code != 200:
|
|
logger.error(f"Failed to fetch members from DAOhaus API: {response.text}")
|
|
return self.get_hardcoded_members()
|
|
|
|
data = response.json()
|
|
|
|
if not data or "members" not in data:
|
|
logger.warning("No members found in DAOhaus API response, falling back to hardcoded list")
|
|
return self.get_hardcoded_members()
|
|
|
|
members_data = data.get("members", [])
|
|
logger.info(f"Found {len(members_data)} members in DAOhaus API response")
|
|
|
|
members = []
|
|
for member in members_data:
|
|
address = member.get("memberAddress")
|
|
if not address:
|
|
continue
|
|
|
|
# Get shares and loot
|
|
shares = str(member.get("shares", 0))
|
|
loot = str(member.get("loot", 0))
|
|
|
|
# Get join date if available
|
|
joined_at = None
|
|
if "createdAt" in member:
|
|
try:
|
|
joined_at = datetime.fromtimestamp(int(member["createdAt"])).isoformat()
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
members.append({
|
|
"address": address,
|
|
"shares": shares,
|
|
"loot": loot,
|
|
"joined_at": joined_at
|
|
})
|
|
|
|
# Update job with success
|
|
self.db.update_scraping_job(
|
|
job_id=job_id,
|
|
status="completed",
|
|
records_processed=len(members_data),
|
|
records_added=len(members)
|
|
)
|
|
|
|
return members
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error scraping DAOhaus UI: {str(e)}")
|
|
return self.get_hardcoded_members()
|
|
|
|
def get_hardcoded_members(self) -> List[Dict[str, Any]]:
|
|
"""
|
|
Get a hardcoded list of Raid Guild members as a fallback.
|
|
|
|
Returns:
|
|
List of dictionaries containing member addresses
|
|
"""
|
|
logger.info("Using hardcoded list of Raid Guild members")
|
|
|
|
# This is a list of known Raid Guild members (as of the script creation)
|
|
raid_guild_members = [
|
|
# Core members
|
|
"0x2e7f4dd3acd226ddae10246a45337f815cf6b3ff", # Raid Guild member
|
|
"0xb5f16bb483e8ce9cc94b19e5e6ebbdcb33a4ae98", # Raid Guild member
|
|
"0x7f73ddcbdcc7d5beb4d4b16dc3c7b6d200532701", # Raid Guild member
|
|
"0x6e7d79db135ddf4cd2612c800ffd5a6c5cc33c93", # Raid Guild member
|
|
|
|
# Members with ENS names
|
|
"0x839395e20bbb182fa440d08f850e6c7a8f6f0780", # griff.eth
|
|
"0x2d4ac9c27ffFCd87D7fA2619F537C7Eb0db96fb7", # decentralizedceo.eth
|
|
"0x58f123BD4261EA25955B362Be57D89F4B6E7110a", # aaronsoskin.eth
|
|
"0x5A6C1AFa7d14FD608af17d7e58e8DB52DF5d66Ea", # terexitarius.eth
|
|
"0x0e707ab69944829ca6377e8F3AEb0c9709b633F7", # duk3duke.eth
|
|
"0x02736d5c8dcea65539993d143A3DE90ceBcA9c3c", # jeffalomaniac.eth
|
|
|
|
# Additional members
|
|
"0x3b687fFc85F172541BfE874CaB5f297DcCcC75E3", # hollyspirit.eth
|
|
"0x7926dad04fE7c482425D784985B5E24aea03C9fF", # eleventhal.eth
|
|
"0x14Ab7AE4fa2820BE8Bc32044Fe5279b56cCBcC34", # onchainmma.eth
|
|
"0x67A16655c1c46f8822726e989751817c49f29054", # manboy.eth
|
|
"0x46704D605748679934E2E913Ec9C0DB8dECC6CaC", # publicmoloch.eth
|
|
"0xd714Dd60e22BbB1cbAFD0e40dE5Cfa7bBDD3F3C8", # auryn.eth
|
|
"0x7136fbDdD4DFfa2369A9283B6E90A040318011Ca", # billw.eth
|
|
"0x516cAfD745Ec780D20f61c0d71fe258eA765222D", # nintynick.eth
|
|
"0x177d9D0Cc4Db65DaC19A3647fA79687eBb976bBf", # positivesumgames.eth
|
|
"0x9672c0e1639F159334Ca1288D4a24DEb02117291", # puppuccino.eth
|
|
"0x2619c649d98DDdDBB0B218823354FE1D41bF5Ce0", # ehcywsivart.eth
|
|
"0x1253594843798Ff0fcd7Fa221B820C2d3cA58FD5", # irlart.eth
|
|
"0x1dF428833f2C9FB1eF098754e5D710432450d706", # 0xjoshua.eth
|
|
"0xd662fA474C0A1346a26374bb4581D1F6D3Fb2d94", # rolf.eth
|
|
"0x8F942ECED007bD3976927B7958B50Df126FEeCb5", # metadreamer.eth
|
|
"0x03F11c7a45BA8219C87f312EEcB07287C2095302", # 0xtangle.eth
|
|
"0xd26a3F686D43f2A62BA9eaE2ff77e9f516d945B9", # vengist.eth
|
|
"0x09988E9AEb8c0B835619305Abfe2cE68FEa17722", # dermot.eth
|
|
"0xCED608Aa29bB92185D9b6340Adcbfa263DAe075b", # dekan.eth
|
|
"0x824959488bA9a9dAB3775451498D732066a4c8F1", # 4d4n.eth
|
|
|
|
# More members
|
|
"0x1C9F765C579F94f6502aCd9fc356171d85a1F8D0", # bitbeckers.eth
|
|
"0xE04885c3f1419C6E8495C33bDCf5F8387cd88846", # skydao.eth
|
|
"0x6FeD46ed75C1165b6bf5bA21f7F507702A2691cB", # boilerhaus.eth
|
|
"0x44905fC26d081A23b0758f17b5CED1821147670b", # chtoli.eth
|
|
"0xA32D31CC8877bB7961D84156EE4dADe6872EBE15", # kushh.eth
|
|
"0xeC9a65D2515A1b4De8497B9c5E43e254b1eBf93a", # launchninja.eth
|
|
"0x5b87C8323352C57Dac33884154aACE8b3D593A07", # old.devfolio.eth
|
|
"0x77b175d193a19378031F4a81393FC0CBD5cF4079", # shingai.eth
|
|
"0x0CF30daf2Fb962Ed1d5D19C97F5f6651F3b691c1", # fishbiscuit.eth
|
|
"0xEC0a73Cc9b682695959611727dA874aFd8440C21", # fahim.eth
|
|
]
|
|
|
|
members = []
|
|
for address in raid_guild_members:
|
|
members.append({
|
|
"address": address,
|
|
"shares": "0", # We don't have this information
|
|
"loot": "0", # We don't have this information
|
|
"joined_at": None # We don't have this information
|
|
})
|
|
|
|
logger.info(f"Found {len(members)} DAO members in hardcoded list")
|
|
return members
|
|
|
|
def process_members(self, members: List[Dict[str, Any]]) -> None:
|
|
"""
|
|
Process the list of members and store in database.
|
|
|
|
Args:
|
|
members: List of dictionaries containing member addresses
|
|
"""
|
|
logger.info(f"Processing {len(members)} members")
|
|
|
|
members_added = 0
|
|
members_updated = 0
|
|
|
|
for member in members:
|
|
address = Web3.to_checksum_address(member["address"])
|
|
joined_at = member.get("joined_at")
|
|
shares = member.get("shares", "0")
|
|
loot = member.get("loot", "0")
|
|
|
|
# Try to resolve ENS name
|
|
ens_name = self.ens_resolver.get_ens_name(address)
|
|
|
|
# Check if contact already exists
|
|
query = 'SELECT id FROM "Contact" WHERE "ethereumAddress" = %(address)s'
|
|
result = self.db.execute_query(query, {"address": address})
|
|
|
|
if result:
|
|
# Contact exists, update it
|
|
contact_id = result[0]["id"]
|
|
if ens_name:
|
|
self.db.update_contact(contact_id, {"ensName": ens_name})
|
|
members_updated += 1
|
|
else:
|
|
# Contact doesn't exist, create it
|
|
contact_id = self.db.upsert_contact(
|
|
ethereum_address=address,
|
|
ens_name=ens_name
|
|
)
|
|
members_added += 1
|
|
|
|
# Add DAO membership
|
|
self.db.add_dao_membership(
|
|
contact_id=contact_id,
|
|
dao_name=self.dao_name,
|
|
dao_type="Moloch",
|
|
joined_at=joined_at
|
|
)
|
|
|
|
# Add a tag for the DAO
|
|
self.db.add_tag_to_contact(
|
|
contact_id=contact_id,
|
|
tag_name=self.dao_name,
|
|
color="#FF5733" # Example color
|
|
)
|
|
|
|
# Add a note with additional information
|
|
note_content = f"{self.dao_name} Membership Information:\n"
|
|
note_content += f"DAO Address: {self.dao_address} (on Gnosis Chain)\n"
|
|
note_content += f"Member Address: {address}\n"
|
|
if ens_name:
|
|
note_content += f"ENS Name: {ens_name}\n"
|
|
if shares != "0":
|
|
note_content += f"Shares: {shares}\n"
|
|
if loot != "0":
|
|
note_content += f"Loot: {loot}\n"
|
|
if joined_at:
|
|
note_content += f"Joined: {joined_at}\n"
|
|
|
|
self.db.add_note_to_contact(contact_id, note_content)
|
|
|
|
# If we have an ENS name, try to get additional profile information
|
|
if ens_name:
|
|
self.ens_resolver.update_contact_from_ens(contact_id, ens_name)
|
|
|
|
# Rate limiting to avoid API throttling
|
|
time.sleep(0.1)
|
|
|
|
logger.info(f"Added {members_added} new contacts and updated {members_updated} existing contacts")
|
|
|
|
def run(self) -> None:
|
|
"""Run the scraper to fetch and process DAO members."""
|
|
members = self.get_dao_members()
|
|
if members:
|
|
self.process_members(members)
|
|
logger.info("DAO members scraping completed successfully")
|
|
else:
|
|
logger.warning("No members found or error occurred")
|
|
|
|
def main():
|
|
"""Main entry point for the script."""
|
|
scraper = RaidGuildScraper()
|
|
scraper.run()
|
|
|
|
if __name__ == "__main__":
|
|
main() |