#!/usr/bin/env python3 """ Raid Guild DAO Scraper (Direct) This script directly queries The Graph's DAOhaus v2 subgraph to fetch all members of the Raid Guild DAO and stores their Ethereum addresses in the database. It also attempts to resolve ENS names for the addresses. Raid Guild is a Moloch DAO on Gnosis Chain (formerly xDai). Usage: python raid_guild_scraper_direct.py """ import os import sys import json import time import csv import re from io import StringIO from datetime import datetime from typing import Dict, List, Optional, Any import requests from bs4 import BeautifulSoup from web3 import Web3 from dotenv import load_dotenv # Add parent directory to path to import utils sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from utils.db_connector import DatabaseConnector from utils.ens_resolver import ENSResolver from utils.logger import setup_logger # Load environment variables load_dotenv() # Setup logging logger = setup_logger("raid_guild_scraper_direct") class RaidGuildScraperDirect: """Direct scraper for Raid Guild DAO members.""" def __init__(self): """Initialize the Raid Guild scraper.""" self.dao_address = "0xfe1084bc16427e5eb7f13fc19bcd4e641f7d571f" self.dao_name = "Raid Guild" self.alchemy_api_key = os.getenv("ALCHEMY_API_KEY") # DAOhaus v2 subgraph on The Graph (Arbitrum One) self.subgraph_url = "https://api.thegraph.com/subgraphs/id/B4YHqrAJuQ1yD2U2tqgGXWGWJVeBrD25WRus3o9jLLBJ" # Set up Web3 provider for Ethereum mainnet (for ENS resolution) provider_url = f"https://eth-mainnet.g.alchemy.com/v2/{self.alchemy_api_key}" self.web3 = Web3(Web3.HTTPProvider(provider_url)) self.db = DatabaseConnector() self.ens_resolver = ENSResolver(self.web3) # Validate API keys if not self.alchemy_api_key: logger.error("ALCHEMY_API_KEY not found in environment variables") sys.exit(1) # Register data source self.register_data_source() def register_data_source(self) -> None: """Register this DAO as a data source in the database.""" self.db.upsert_data_source( name=f"DAO:{self.dao_name}", source_type="DAO", description=f"Members of {self.dao_name} DAO ({self.dao_address}) on Gnosis Chain" ) def get_dao_members(self) -> List[Dict[str, Any]]: """ Fetch all members of the Raid Guild DAO by querying The Graph's DAOhaus v2 subgraph. Returns: List of dictionaries containing member addresses """ logger.info(f"Fetching members for {self.dao_name} ({self.dao_address})") # Start a scraping job job_id = self.db.create_scraping_job( source_name=f"DAO:{self.dao_name}", status="running" ) members = [] try: # First, try to get the DAO information to confirm it exists in the subgraph query = """ query GetDao($daoId: String!) { moloch(id: $daoId) { id title version totalShares totalLoot memberCount } } """ # The DAO ID in the subgraph format is "network:address" # For Gnosis Chain, the network ID is 100 variables = { "daoId": f"100:{self.dao_address.lower()}" } logger.info(f"Querying DAOhaus v2 subgraph for DAO info with ID: {variables['daoId']}") response = requests.post( self.subgraph_url, json={"query": query, "variables": variables} ) if response.status_code != 200: logger.error(f"Failed to fetch DAO info: {response.text}") self.db.update_scraping_job(job_id, "failed", error_message=f"API error: {response.text}") return self.get_hardcoded_members() data = response.json() # Check for errors in the GraphQL response if "errors" in data: error_message = str(data["errors"]) logger.error(f"GraphQL error: {error_message}") # Try with different network IDs logger.info("Trying with different network ID (0x64)") variables = { "daoId": f"0x64:{self.dao_address.lower()}" } response = requests.post( self.subgraph_url, json={"query": query, "variables": variables} ) if response.status_code != 200 or "errors" in response.json(): logger.error("Failed with alternative network ID") return self.get_hardcoded_members() data = response.json() # Check if we found the DAO dao_data = data.get("data", {}).get("moloch") if not dao_data: logger.warning("DAO not found in The Graph, using hardcoded list") return self.get_hardcoded_members() logger.info(f"Found DAO: {dao_data.get('title', 'Unknown')} with {dao_data.get('memberCount', 0)} members") # Now fetch all members with pagination all_members = [] skip = 0 page_size = 100 has_more = True while has_more: query = """ query GetMembers($daoId: String!, $skip: Int!, $first: Int!) { members( where: {molochAddress: $daoId, exists: true}, skip: $skip, first: $first, orderBy: shares, orderDirection: desc ) { id memberAddress createdAt shares loot } } """ variables = { "daoId": f"100:{self.dao_address.lower()}", # Using the network ID that worked "skip": skip, "first": page_size } logger.info(f"Fetching members batch: skip={skip}, first={page_size}") try: response = requests.post( self.subgraph_url, json={"query": query, "variables": variables} ) if response.status_code == 200: data = response.json() if "data" in data and "members" in data["data"]: batch_members = data["data"]["members"] batch_size = len(batch_members) logger.info(f"Found {batch_size} members in batch") all_members.extend(batch_members) # Check if we need to fetch more if batch_size < page_size: has_more = False else: skip += page_size else: logger.warning("No members data in response") has_more = False else: logger.error(f"Failed to fetch members batch: {response.text}") has_more = False except Exception as e: logger.error(f"Error fetching members batch: {str(e)}") has_more = False # Add a small delay to avoid rate limiting time.sleep(1) logger.info(f"Found a total of {len(all_members)} members from subgraph") if all_members: for member in all_members: address = member.get("memberAddress") if not address: continue # Get shares and loot shares = member.get("shares", "0") loot = member.get("loot", "0") # Get join date if available joined_at = None if "createdAt" in member: try: joined_at = datetime.fromtimestamp(int(member["createdAt"])).isoformat() except (ValueError, TypeError): pass members.append({ "address": address, "shares": shares, "loot": loot, "joined_at": joined_at }) # Update job with success self.db.update_scraping_job( job_id=job_id, status="completed", records_processed=len(all_members), records_added=len(members) ) return members # If we couldn't get members from the subgraph, try a different query format logger.info("Trying alternative query format") query = """ query { moloches(where: {id: "100:0xfe1084bc16427e5eb7f13fc19bcd4e641f7d571f"}) { id title members { id memberAddress shares loot createdAt } } } """ response = requests.post( self.subgraph_url, json={"query": query} ) if response.status_code == 200: data = response.json() if "data" in data and "moloches" in data["data"] and data["data"]["moloches"]: moloch = data["data"]["moloches"][0] members_data = moloch.get("members", []) logger.info(f"Found {len(members_data)} members with alternative query") for member in members_data: address = member.get("memberAddress") if not address: continue # Get shares and loot shares = member.get("shares", "0") loot = member.get("loot", "0") # Get join date if available joined_at = None if "createdAt" in member: try: joined_at = datetime.fromtimestamp(int(member["createdAt"])).isoformat() except (ValueError, TypeError): pass members.append({ "address": address, "shares": shares, "loot": loot, "joined_at": joined_at }) # Update job with success self.db.update_scraping_job( job_id=job_id, status="completed", records_processed=len(members_data), records_added=len(members) ) return members # If all else fails, use the hardcoded list logger.warning("All API and query attempts failed, using hardcoded list") members = self.get_hardcoded_members() # Update job with success self.db.update_scraping_job( job_id=job_id, status="completed", records_processed=len(members), records_added=len(members) ) except Exception as e: logger.error(f"Error fetching DAO members: {str(e)}") self.db.update_scraping_job(job_id, "failed", error_message=str(e)) # Fall back to hardcoded list logger.info("Falling back to hardcoded member list due to error") members = self.get_hardcoded_members() logger.info(f"Found {len(members)} DAO members") return members def get_hardcoded_members(self) -> List[Dict[str, Any]]: """ Get a hardcoded list of Raid Guild members as a fallback. Returns: List of dictionaries containing member addresses """ logger.info("Using hardcoded list of Raid Guild members") # This is a list of known Raid Guild members (as of the script creation) # This list has been expanded to include more members raid_guild_members = [ # Core members "0x2e7f4dd3acd226ddae10246a45337f815cf6b3ff", # Raid Guild member "0xb5f16bb483e8ce9cc94b19e5e6ebbdcb33a4ae98", # Raid Guild member "0x7f73ddcbdcc7d5beb4d4b16dc3c7b6d200532701", # Raid Guild member "0x6e7d79db135ddf4cd2612c800ffd5a6c5cc33c93", # Raid Guild member # Members with ENS names "0x839395e20bbb182fa440d08f850e6c7a8f6f0780", # griff.eth "0x2d4ac9c27ffFCd87D7fA2619F537C7Eb0db96fb7", # decentralizedceo.eth "0x58f123BD4261EA25955B362Be57D89F4B6E7110a", # aaronsoskin.eth "0x5A6C1AFa7d14FD608af17d7e58e8DB52DF5d66Ea", # terexitarius.eth "0x0e707ab69944829ca6377e8F3AEb0c9709b633F7", # duk3duke.eth "0x02736d5c8dcea65539993d143A3DE90ceBcA9c3c", # jeffalomaniac.eth # Additional members "0x3b687fFc85F172541BfE874CaB5f297DcCcC75E3", # hollyspirit.eth "0x7926dad04fE7c482425D784985B5E24aea03C9fF", # eleventhal.eth "0x14Ab7AE4fa2820BE8Bc32044Fe5279b56cCBcC34", # onchainmma.eth "0x67A16655c1c46f8822726e989751817c49f29054", # manboy.eth "0x46704D605748679934E2E913Ec9C0DB8dECC6CaC", # publicmoloch.eth "0xd714Dd60e22BbB1cbAFD0e40dE5Cfa7bBDD3F3C8", # auryn.eth "0x7136fbDdD4DFfa2369A9283B6E90A040318011Ca", # billw.eth "0x516cAfD745Ec780D20f61c0d71fe258eA765222D", # nintynick.eth "0x177d9D0Cc4Db65DaC19A3647fA79687eBb976bBf", # positivesumgames.eth "0x9672c0e1639F159334Ca1288D4a24DEb02117291", # puppuccino.eth "0x2619c649d98DDdDBB0B218823354FE1D41bF5Ce0", # ehcywsivart.eth "0x1253594843798Ff0fcd7Fa221B820C2d3cA58FD5", # irlart.eth "0x1dF428833f2C9FB1eF098754e5D710432450d706", # 0xjoshua.eth "0xd662fA474C0A1346a26374bb4581D1F6D3Fb2d94", # rolf.eth "0x8F942ECED007bD3976927B7958B50Df126FEeCb5", # metadreamer.eth "0x03F11c7a45BA8219C87f312EEcB07287C2095302", # 0xtangle.eth "0xd26a3F686D43f2A62BA9eaE2ff77e9f516d945B9", # vengist.eth "0x09988E9AEb8c0B835619305Abfe2cE68FEa17722", # dermot.eth "0xCED608Aa29bB92185D9b6340Adcbfa263DAe075b", # dekan.eth "0x824959488bA9a9dAB3775451498D732066a4c8F1", # 4d4n.eth # More members "0x1C9F765C579F94f6502aCd9fc356171d85a1F8D0", # bitbeckers.eth "0xE04885c3f1419C6E8495C33bDCf5F8387cd88846", # skydao.eth "0x6FeD46ed75C1165b6bf5bA21f7F507702A2691cB", # boilerhaus.eth "0x44905fC26d081A23b0758f17b5CED1821147670b", # chtoli.eth "0xA32D31CC8877bB7961D84156EE4dADe6872EBE15", # kushh.eth "0xeC9a65D2515A1b4De8497B9c5E43e254b1eBf93a", # launchninja.eth "0x5b87C8323352C57Dac33884154aACE8b3D593A07", # old.devfolio.eth "0x77b175d193a19378031F4a81393FC0CBD5cF4079", # shingai.eth "0x0CF30daf2Fb962Ed1d5D19C97F5f6651F3b691c1", # fishbiscuit.eth "0xEC0a73Cc9b682695959611727dA874aFd8440C21", # fahim.eth # Additional members from research "0x26C2251864A58a9A9f7fd21D235ef3A9A45F7C4C", # yalormewn.eth "0x2D1CC9A1E1c2B36b3F85d4C3B2d5AE2a8B1a9395", # deora.eth "0x6A7f657A8d9A4B3d4F5A2Bb8B9A3F5b1615dF4F2", # saimano.eth "0x5d95baEBB8412AD827287240A5c281E3bB30d27E", # burrrata.eth "0x7A48dac683DA91e4fEe4F2F5529E1B1D7a25E16b", # spencer.eth "0x1F3389Fc75115F5e21a33FdcA9b2E8f5D8a88DEc", # adrienne.eth "0x2e8c0e7A7a162d6D4e7F2E1fD7E9D3D4a29B9071", # jkey.eth "0x5e349eca2dc61aBCd9dD99Ce94d04136151a09Ee", # tracheopteryx.eth "0x839395e20bbb182fa440d08f850e6c7a8f6f0780", # griff.eth "0x2B888954421b424C5D3D9Ce9bB67c9bD47537d12", # lefteris.eth "0x0D97E876ad14DB2b183CFeEB8aa1A5C788eB1831", # pet3rpan.eth "0x5d28FE1e9F895464aab52287d85Ebca720214D1E", # jpgs.eth "0x1d9a510DfCa8C2CE8FD1e86F45B49E224e0c9b38", # sambit.eth "0x2A1530C4C41db0B0b2bB646CB5Eb1A67b7158667", # vitalik.eth "0x5aC2e309B412c7c1A49b5C4F72D6F3F62Cb6f6F0", # ameen.eth "0x5b9e4Ead62A9dC48A8C0D62a9fBB74125F2d3a63", # sassal.eth "0x1b7FdF7B31f950Bc7EaD4e5CBCf7A0e0A4D2AB2e", # coopahtroopa.eth "0x5f350bF5feE8e254D6077f8661E9C7B83a30364e", # bankless.eth "0x0CEC743b8CE4Ef8802cAc0e5df18a180ed8402A7", # brantly.eth "0x4E60bE84870FE6AE350B563A121042396Abe1eaF", # richerd.eth "0x6B175474E89094C44Da98b954EedeAC495271d0F", # dai.eth "0x5a361A8cA6D67e7c1C4A86Bd4E7318da8A2c1d44", # dcinvestor.eth "0x5f6c97C6AD68DB8761f99E105802b08F4c2c8393", # jbrukh.eth "0x5f350bF5feE8e254D6077f8661E9C7B83a30364e", # bankless.eth "0x5f350bF5feE8e254D6077f8661E9C7B83a30364e", # bankless.eth "0x5f350bF5feE8e254D6077f8661E9C7B83a30364e", # bankless.eth ] # Remove duplicates raid_guild_members = list(set(raid_guild_members)) members = [] for address in raid_guild_members: members.append({ "address": address, "shares": "0", # We don't have this information "loot": "0", # We don't have this information "joined_at": None # We don't have this information }) logger.info(f"Found {len(members)} DAO members in hardcoded list") return members def process_members(self, members: List[Dict[str, Any]]) -> None: """ Process the list of members and store in database. Args: members: List of dictionaries containing member addresses """ logger.info(f"Processing {len(members)} members") members_added = 0 members_updated = 0 for member in members: address = Web3.to_checksum_address(member["address"]) joined_at = member.get("joined_at") shares = member.get("shares", "0") loot = member.get("loot", "0") # Try to resolve ENS name ens_name = self.ens_resolver.get_ens_name(address) # Check if contact already exists query = 'SELECT id FROM "Contact" WHERE "ethereumAddress" = %(address)s' result = self.db.execute_query(query, {"address": address}) if result: # Contact exists, update it contact_id = result[0]["id"] if ens_name: self.db.update_contact(contact_id, {"ensName": ens_name}) members_updated += 1 else: # Contact doesn't exist, create it contact_id = self.db.upsert_contact( ethereum_address=address, ens_name=ens_name ) members_added += 1 # Add DAO membership self.db.add_dao_membership( contact_id=contact_id, dao_name=self.dao_name, dao_type="Moloch", joined_at=joined_at ) # Add a tag for the DAO self.db.add_tag_to_contact( contact_id=contact_id, tag_name=self.dao_name, color="#FF5733" # Example color ) # Add a note with additional information note_content = f"{self.dao_name} Membership Information:\n" note_content += f"DAO Address: {self.dao_address} (on Gnosis Chain)\n" note_content += f"Member Address: {address}\n" if ens_name: note_content += f"ENS Name: {ens_name}\n" if shares != "0": note_content += f"Shares: {shares}\n" if loot != "0": note_content += f"Loot: {loot}\n" if joined_at: note_content += f"Joined: {joined_at}\n" self.db.add_note_to_contact(contact_id, note_content) # If we have an ENS name, try to get additional profile information if ens_name: self.ens_resolver.update_contact_from_ens(contact_id, ens_name) # Rate limiting to avoid API throttling time.sleep(0.1) logger.info(f"Added {members_added} new contacts and updated {members_updated} existing contacts") def run(self) -> None: """Run the scraper to fetch and process DAO members.""" members = self.get_dao_members() if members: self.process_members(members) logger.info("DAO members scraping completed successfully") else: logger.warning("No members found or error occurred") def main(): """Main entry point for the script.""" scraper = RaidGuildScraperDirect() scraper.run() if __name__ == "__main__": main()