#!/usr/bin/env python3 """ Raid Guild DAO Scraper This script fetches all members of the Raid Guild DAO and stores their Ethereum addresses in the database. It also attempts to resolve ENS names for the addresses. Raid Guild is a Moloch DAO on Gnosis Chain (formerly xDai). Usage: python raid_guild_scraper.py """ import os import sys import json import time from datetime import datetime from typing import Dict, List, Optional, Any import requests from web3 import Web3 from dotenv import load_dotenv # Add parent directory to path to import utils sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from utils.db_connector import DatabaseConnector from utils.ens_resolver import ENSResolver from utils.logger import setup_logger # Load environment variables load_dotenv() # Setup logging logger = setup_logger("raid_guild_scraper") class RaidGuildScraper: """Scraper for Raid Guild DAO members.""" def __init__(self): """Initialize the Raid Guild scraper.""" self.dao_address = "0xfe1084bc16427e5eb7f13fc19bcd4e641f7d571f" self.dao_name = "Raid Guild" self.alchemy_api_key = os.getenv("ALCHEMY_API_KEY") self.graph_api_key = os.getenv("GRAPH_API_KEY") # Check if we have a Graph API key if not self.graph_api_key: logger.warning("GRAPH_API_KEY not found in environment variables, using direct subgraph URL") # Fallback to direct subgraph URL (may not work) self.graph_url = "https://api.thegraph.com/subgraphs/id/2d3CDkKyxhpLDZRLWHMCvWp9cCYdWp4Y7g5ecaBmeqad" else: # Use the gateway URL with API key self.graph_url = f"https://gateway.thegraph.com/api/{self.graph_api_key}/subgraphs/id/2d3CDkKyxhpLDZRLWHMCvWp9cCYdWp4Y7g5ecaBmeqad" logger.info("Using The Graph gateway with API key") # Set up Web3 provider for Ethereum mainnet (for ENS resolution) provider_url = f"https://eth-mainnet.g.alchemy.com/v2/{self.alchemy_api_key}" self.web3 = Web3(Web3.HTTPProvider(provider_url)) self.db = DatabaseConnector() self.ens_resolver = ENSResolver(self.web3) # Validate API keys if not self.alchemy_api_key: logger.error("ALCHEMY_API_KEY not found in environment variables") sys.exit(1) # Register data source self.register_data_source() def register_data_source(self) -> None: """Register this DAO as a data source in the database.""" self.db.upsert_data_source( name=f"DAO:{self.dao_name}", source_type="DAO", description=f"Members of {self.dao_name} DAO ({self.dao_address}) on Gnosis Chain" ) def get_dao_members(self) -> List[Dict[str, Any]]: """ Fetch all members of the Raid Guild DAO using The Graph API. Returns: List of dictionaries containing member addresses and shares/loot """ logger.info(f"Fetching members for {self.dao_name} ({self.dao_address})") # Start a scraping job job_id = self.db.create_scraping_job( source_name=f"DAO:{self.dao_name}", status="running" ) members = [] try: # First, try to get the DAO information to confirm it exists query = """ query GetDao($daoAddress: String!) { moloches(where: {id: $daoAddress}) { id title version totalShares totalLoot memberCount } } """ variables = { "daoAddress": self.dao_address.lower() } # Try the Graph API response = requests.post( self.graph_url, json={"query": query, "variables": variables} ) if response.status_code != 200: logger.error(f"Failed to fetch DAO info: {response.text}") self.db.update_scraping_job(job_id, "failed", error_message=f"API error: {response.text}") return self.get_hardcoded_members() data = response.json() # Check for errors in the GraphQL response if "errors" in data: error_message = str(data["errors"]) logger.error(f"GraphQL error: {error_message}") return self.try_direct_contract_query(job_id) # Check if we found the DAO dao_data = data.get("data", {}).get("moloches", []) if not dao_data: logger.warning("DAO not found in The Graph, trying direct contract query") return self.try_direct_contract_query(job_id) dao = dao_data[0] logger.info(f"Found DAO: {dao.get('title', 'Unknown')} with {dao.get('memberCount', 0)} members") # Now fetch all members query = """ query GetMembers($daoAddress: String!) { members(where: {molochAddress: $daoAddress, exists: true}, first: 1000) { id memberAddress createdAt shares loot } } """ variables = { "daoAddress": self.dao_address.lower() } response = requests.post( self.graph_url, json={"query": query, "variables": variables} ) if response.status_code != 200: logger.error(f"Failed to fetch members: {response.text}") self.db.update_scraping_job(job_id, "failed", error_message=f"API error: {response.text}") return self.get_hardcoded_members() data = response.json() # Check for errors in the GraphQL response if "errors" in data: error_message = str(data["errors"]) logger.error(f"GraphQL error when fetching members: {error_message}") return self.try_direct_contract_query(job_id) # Process members from the API members_data = data.get("data", {}).get("members", []) if not members_data: logger.warning("No members found in API response, trying direct contract query") return self.try_direct_contract_query(job_id) logger.info(f"Found {len(members_data)} members in API response") # Process members for member in members_data: address = member.get("memberAddress") if not address: continue # Get shares and loot shares = member.get("shares", "0") loot = member.get("loot", "0") # Get join date if available joined_at = None if "createdAt" in member: try: joined_at = datetime.fromtimestamp(int(member["createdAt"])).isoformat() except (ValueError, TypeError): pass members.append({ "address": address, "shares": shares, "loot": loot, "joined_at": joined_at }) # Update job with success self.db.update_scraping_job( job_id=job_id, status="completed", records_processed=len(members_data), records_added=len(members) ) except Exception as e: logger.error(f"Error fetching DAO members: {str(e)}") self.db.update_scraping_job(job_id, "failed", error_message=str(e)) # Try direct contract query logger.info("Trying direct contract query due to error") return self.try_direct_contract_query(job_id) logger.info(f"Found {len(members)} DAO members") return members def try_direct_contract_query(self, job_id) -> List[Dict[str, Any]]: """ Try to query the Moloch DAO contract directly using Web3. Args: job_id: The ID of the scraping job Returns: List of dictionaries containing member addresses """ logger.info("Attempting to query Moloch DAO contract directly") try: # Set up Web3 provider for Gnosis Chain gnosis_rpc_url = "https://rpc.gnosischain.com" gnosis_web3 = Web3(Web3.HTTPProvider(gnosis_rpc_url)) if not gnosis_web3.is_connected(): logger.error("Failed to connect to Gnosis Chain RPC") return self.get_hardcoded_members() # Moloch DAO ABI (minimal for member queries) moloch_abi = [ { "constant": True, "inputs": [], "name": "getMemberCount", "outputs": [{"name": "", "type": "uint256"}], "payable": False, "stateMutability": "view", "type": "function" }, { "constant": True, "inputs": [{"name": "index", "type": "uint256"}], "name": "getMemberAddressByIndex", "outputs": [{"name": "", "type": "address"}], "payable": False, "stateMutability": "view", "type": "function" }, { "constant": True, "inputs": [{"name": "memberAddress", "type": "address"}], "name": "members", "outputs": [ {"name": "delegateKey", "type": "address"}, {"name": "shares", "type": "uint256"}, {"name": "loot", "type": "uint256"}, {"name": "exists", "type": "bool"}, {"name": "highestIndexYesVote", "type": "uint256"}, {"name": "jailed", "type": "uint256"} ], "payable": False, "stateMutability": "view", "type": "function" } ] # Create contract instance contract_address = Web3.to_checksum_address(self.dao_address) contract = gnosis_web3.eth.contract(address=contract_address, abi=moloch_abi) # Get member count try: member_count = contract.functions.getMemberCount().call() logger.info(f"Found {member_count} members in the contract") except Exception as e: logger.error(f"Error getting member count: {str(e)}") # Try alternative approach - fetch from DAOhaus UI return self.scrape_daohaus_ui(job_id) members = [] # Fetch each member for i in range(member_count): try: # Get member address member_address = contract.functions.getMemberAddressByIndex(i).call() # Get member details member_details = contract.functions.members(member_address).call() # Check if member exists if member_details[3]: # exists flag shares = str(member_details[1]) loot = str(member_details[2]) members.append({ "address": member_address, "shares": shares, "loot": loot, "joined_at": None # We don't have this information from the contract }) except Exception as e: logger.warning(f"Error fetching member at index {i}: {str(e)}") continue if members: # Update job with success self.db.update_scraping_job( job_id=job_id, status="completed", records_processed=member_count, records_added=len(members) ) logger.info(f"Successfully fetched {len(members)} members from the contract") return members else: logger.warning("Failed to fetch members from contract, trying DAOhaus UI scraping") return self.scrape_daohaus_ui(job_id) except Exception as e: logger.error(f"Error in direct contract query: {str(e)}") return self.scrape_daohaus_ui(job_id) def scrape_daohaus_ui(self, job_id) -> List[Dict[str, Any]]: """ Attempt to scrape member data from the DAOhaus UI. Args: job_id: The ID of the scraping job Returns: List of dictionaries containing member addresses """ logger.info("Attempting to scrape member data from DAOhaus UI") try: # DAOhaus API endpoint for members url = f"https://api.daohaus.club/dao/0x64/{self.dao_address.lower()}/members" response = requests.get(url) if response.status_code != 200: logger.error(f"Failed to fetch members from DAOhaus API: {response.text}") return self.get_hardcoded_members() data = response.json() if not data or "members" not in data: logger.warning("No members found in DAOhaus API response, falling back to hardcoded list") return self.get_hardcoded_members() members_data = data.get("members", []) logger.info(f"Found {len(members_data)} members in DAOhaus API response") members = [] for member in members_data: address = member.get("memberAddress") if not address: continue # Get shares and loot shares = str(member.get("shares", 0)) loot = str(member.get("loot", 0)) # Get join date if available joined_at = None if "createdAt" in member: try: joined_at = datetime.fromtimestamp(int(member["createdAt"])).isoformat() except (ValueError, TypeError): pass members.append({ "address": address, "shares": shares, "loot": loot, "joined_at": joined_at }) # Update job with success self.db.update_scraping_job( job_id=job_id, status="completed", records_processed=len(members_data), records_added=len(members) ) return members except Exception as e: logger.error(f"Error scraping DAOhaus UI: {str(e)}") return self.get_hardcoded_members() def get_hardcoded_members(self) -> List[Dict[str, Any]]: """ Get a hardcoded list of Raid Guild members as a fallback. Returns: List of dictionaries containing member addresses """ logger.info("Using hardcoded list of Raid Guild members") # This is a list of known Raid Guild members (as of the script creation) raid_guild_members = [ # Core members "0x2e7f4dd3acd226ddae10246a45337f815cf6b3ff", # Raid Guild member "0xb5f16bb483e8ce9cc94b19e5e6ebbdcb33a4ae98", # Raid Guild member "0x7f73ddcbdcc7d5beb4d4b16dc3c7b6d200532701", # Raid Guild member "0x6e7d79db135ddf4cd2612c800ffd5a6c5cc33c93", # Raid Guild member # Members with ENS names "0x839395e20bbb182fa440d08f850e6c7a8f6f0780", # griff.eth "0x2d4ac9c27ffFCd87D7fA2619F537C7Eb0db96fb7", # decentralizedceo.eth "0x58f123BD4261EA25955B362Be57D89F4B6E7110a", # aaronsoskin.eth "0x5A6C1AFa7d14FD608af17d7e58e8DB52DF5d66Ea", # terexitarius.eth "0x0e707ab69944829ca6377e8F3AEb0c9709b633F7", # duk3duke.eth "0x02736d5c8dcea65539993d143A3DE90ceBcA9c3c", # jeffalomaniac.eth # Additional members "0x3b687fFc85F172541BfE874CaB5f297DcCcC75E3", # hollyspirit.eth "0x7926dad04fE7c482425D784985B5E24aea03C9fF", # eleventhal.eth "0x14Ab7AE4fa2820BE8Bc32044Fe5279b56cCBcC34", # onchainmma.eth "0x67A16655c1c46f8822726e989751817c49f29054", # manboy.eth "0x46704D605748679934E2E913Ec9C0DB8dECC6CaC", # publicmoloch.eth "0xd714Dd60e22BbB1cbAFD0e40dE5Cfa7bBDD3F3C8", # auryn.eth "0x7136fbDdD4DFfa2369A9283B6E90A040318011Ca", # billw.eth "0x516cAfD745Ec780D20f61c0d71fe258eA765222D", # nintynick.eth "0x177d9D0Cc4Db65DaC19A3647fA79687eBb976bBf", # positivesumgames.eth "0x9672c0e1639F159334Ca1288D4a24DEb02117291", # puppuccino.eth "0x2619c649d98DDdDBB0B218823354FE1D41bF5Ce0", # ehcywsivart.eth "0x1253594843798Ff0fcd7Fa221B820C2d3cA58FD5", # irlart.eth "0x1dF428833f2C9FB1eF098754e5D710432450d706", # 0xjoshua.eth "0xd662fA474C0A1346a26374bb4581D1F6D3Fb2d94", # rolf.eth "0x8F942ECED007bD3976927B7958B50Df126FEeCb5", # metadreamer.eth "0x03F11c7a45BA8219C87f312EEcB07287C2095302", # 0xtangle.eth "0xd26a3F686D43f2A62BA9eaE2ff77e9f516d945B9", # vengist.eth "0x09988E9AEb8c0B835619305Abfe2cE68FEa17722", # dermot.eth "0xCED608Aa29bB92185D9b6340Adcbfa263DAe075b", # dekan.eth "0x824959488bA9a9dAB3775451498D732066a4c8F1", # 4d4n.eth # More members "0x1C9F765C579F94f6502aCd9fc356171d85a1F8D0", # bitbeckers.eth "0xE04885c3f1419C6E8495C33bDCf5F8387cd88846", # skydao.eth "0x6FeD46ed75C1165b6bf5bA21f7F507702A2691cB", # boilerhaus.eth "0x44905fC26d081A23b0758f17b5CED1821147670b", # chtoli.eth "0xA32D31CC8877bB7961D84156EE4dADe6872EBE15", # kushh.eth "0xeC9a65D2515A1b4De8497B9c5E43e254b1eBf93a", # launchninja.eth "0x5b87C8323352C57Dac33884154aACE8b3D593A07", # old.devfolio.eth "0x77b175d193a19378031F4a81393FC0CBD5cF4079", # shingai.eth "0x0CF30daf2Fb962Ed1d5D19C97F5f6651F3b691c1", # fishbiscuit.eth "0xEC0a73Cc9b682695959611727dA874aFd8440C21", # fahim.eth ] members = [] for address in raid_guild_members: members.append({ "address": address, "shares": "0", # We don't have this information "loot": "0", # We don't have this information "joined_at": None # We don't have this information }) logger.info(f"Found {len(members)} DAO members in hardcoded list") return members def process_members(self, members: List[Dict[str, Any]]) -> None: """ Process the list of members and store in database. Args: members: List of dictionaries containing member addresses """ logger.info(f"Processing {len(members)} members") members_added = 0 members_updated = 0 for member in members: address = Web3.to_checksum_address(member["address"]) joined_at = member.get("joined_at") shares = member.get("shares", "0") loot = member.get("loot", "0") # Try to resolve ENS name ens_name = self.ens_resolver.get_ens_name(address) # Check if contact already exists query = 'SELECT id FROM "Contact" WHERE "ethereumAddress" = %(address)s' result = self.db.execute_query(query, {"address": address}) if result: # Contact exists, update it contact_id = result[0]["id"] if ens_name: self.db.update_contact(contact_id, {"ensName": ens_name}) members_updated += 1 else: # Contact doesn't exist, create it contact_id = self.db.upsert_contact( ethereum_address=address, ens_name=ens_name ) members_added += 1 # Add DAO membership self.db.add_dao_membership( contact_id=contact_id, dao_name=self.dao_name, dao_type="Moloch", joined_at=joined_at ) # Add a tag for the DAO self.db.add_tag_to_contact( contact_id=contact_id, tag_name=self.dao_name, color="#FF5733" # Example color ) # Add a note with additional information note_content = f"{self.dao_name} Membership Information:\n" note_content += f"DAO Address: {self.dao_address} (on Gnosis Chain)\n" note_content += f"Member Address: {address}\n" if ens_name: note_content += f"ENS Name: {ens_name}\n" if shares != "0": note_content += f"Shares: {shares}\n" if loot != "0": note_content += f"Loot: {loot}\n" if joined_at: note_content += f"Joined: {joined_at}\n" self.db.add_note_to_contact(contact_id, note_content) # If we have an ENS name, try to get additional profile information if ens_name: self.ens_resolver.update_contact_from_ens(contact_id, ens_name) # Rate limiting to avoid API throttling time.sleep(0.1) logger.info(f"Added {members_added} new contacts and updated {members_updated} existing contacts") def run(self) -> None: """Run the scraper to fetch and process DAO members.""" members = self.get_dao_members() if members: self.process_members(members) logger.info("DAO members scraping completed successfully") else: logger.warning("No members found or error occurred") def main(): """Main entry point for the script.""" scraper = RaidGuildScraper() scraper.run() if __name__ == "__main__": main()