#!/usr/bin/env python3 """ Moloch DAO Scraper This script fetches all members of a specific Moloch DAO and stores their Ethereum addresses in the database. It also attempts to resolve ENS names for the addresses. Usage: python moloch_dao_scraper.py --dao-address 0xfe1084bc16427e5eb7f13fc19bcd4e641f7d571f --dao-name "Raid Guild" --network 0x64 """ import os import sys import argparse import json import time from datetime import datetime from typing import Dict, List, Optional, Any import requests from web3 import Web3 from dotenv import load_dotenv # Add parent directory to path to import utils sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from utils.db_connector import DatabaseConnector from utils.ens_resolver import ENSResolver from utils.logger import setup_logger # Load environment variables load_dotenv() # Setup logging logger = setup_logger("moloch_dao_scraper") # Moloch DAO ABI (partial, only the functions we need) MOLOCH_DAO_ABI = [ { "constant": True, "inputs": [], "name": "memberCount", "outputs": [{"name": "", "type": "uint256"}], "payable": False, "stateMutability": "view", "type": "function" }, { "constant": True, "inputs": [{"name": "", "type": "address"}], "name": "members", "outputs": [ {"name": "delegateKey", "type": "address"}, {"name": "shares", "type": "uint256"}, {"name": "loot", "type": "uint256"}, {"name": "exists", "type": "bool"}, {"name": "highestIndexYesVote", "type": "uint256"}, {"name": "jailed", "type": "uint256"} ], "payable": False, "stateMutability": "view", "type": "function" }, { "constant": True, "inputs": [{"name": "", "type": "uint256"}], "name": "memberAddressByIndex", "outputs": [{"name": "", "type": "address"}], "payable": False, "stateMutability": "view", "type": "function" } ] class MolochDAOScraper: """Scraper for Moloch DAO members.""" def __init__(self, dao_address: str, dao_name: str, network: str = "0x1"): """ Initialize the Moloch DAO scraper. Args: dao_address: Ethereum address of the DAO contract dao_name: Name of the DAO network: Network ID (default: "0x1" for Ethereum mainnet, "0x64" for Gnosis Chain) """ self.dao_address = Web3.to_checksum_address(dao_address) self.dao_name = dao_name self.network = network self.alchemy_api_key = os.getenv("ALCHEMY_API_KEY") # Set up Web3 provider based on network if network == "0x1": # Ethereum mainnet provider_url = f"https://eth-mainnet.g.alchemy.com/v2/{self.alchemy_api_key}" elif network == "0x64": # Gnosis Chain (xDai) provider_url = "https://rpc.gnosischain.com" else: logger.error(f"Unsupported network: {network}") sys.exit(1) self.web3 = Web3(Web3.HTTPProvider(provider_url)) self.db = DatabaseConnector() self.ens_resolver = ENSResolver(self.web3) # Initialize the DAO contract self.dao_contract = self.web3.eth.contract( address=self.dao_address, abi=MOLOCH_DAO_ABI ) # Validate API keys if not self.alchemy_api_key: logger.error("ALCHEMY_API_KEY not found in environment variables") sys.exit(1) # Register data source self.register_data_source() def register_data_source(self) -> None: """Register this DAO as a data source in the database.""" self.db.upsert_data_source( name=f"DAO:{self.dao_name}", source_type="DAO", description=f"Members of {self.dao_name} DAO ({self.dao_address})" ) def get_dao_members(self) -> List[Dict[str, Any]]: """ Fetch all members of the DAO by directly querying the contract. Returns: List of dictionaries containing member addresses and shares/loot """ logger.info(f"Fetching members for {self.dao_name} ({self.dao_address})") # Start a scraping job job_id = self.db.create_scraping_job( source_name=f"DAO:{self.dao_name}", status="running" ) members = [] try: # Get the total number of members try: member_count = self.dao_contract.functions.memberCount().call() logger.info(f"Member count from contract: {member_count}") except Exception as e: logger.warning(f"Could not get member count: {str(e)}") # If memberCount function is not available, we'll try a different approach member_count = 0 index = 0 while True: try: # Try to get member address at index address = self.dao_contract.functions.memberAddressByIndex(index).call() if address != "0x0000000000000000000000000000000000000000": member_count += 1 index += 1 else: break except Exception: # If we get an error, we've reached the end of the list break logger.info(f"Estimated member count: {member_count}") # Fetch all member addresses member_addresses = [] for i in range(member_count): try: address = self.dao_contract.functions.memberAddressByIndex(i).call() if address != "0x0000000000000000000000000000000000000000": member_addresses.append(address) except Exception as e: logger.warning(f"Error getting member at index {i}: {str(e)}") continue logger.info(f"Found {len(member_addresses)} member addresses") # Get member details for each address for address in member_addresses: try: member_data = self.dao_contract.functions.members(address).call() # Check if the member exists if not member_data[3]: # exists flag continue members.append({ "address": address, "shares": str(member_data[1]), # shares "loot": str(member_data[2]), # loot "joined_at": None # We don't have this information from the contract }) except Exception as e: logger.warning(f"Error getting member data for {address}: {str(e)}") continue # Update job with success self.db.update_scraping_job( job_id=job_id, status="completed", records_processed=len(member_addresses), records_added=len(members) ) except Exception as e: logger.error(f"Error fetching DAO members: {str(e)}") self.db.update_scraping_job(job_id, "failed", error_message=str(e)) return [] logger.info(f"Found {len(members)} DAO members") return members def process_members(self, members: List[Dict[str, Any]]) -> None: """ Process the list of members and store in database. Args: members: List of dictionaries containing member addresses and shares/loot """ logger.info(f"Processing {len(members)} members") members_added = 0 members_updated = 0 for member in members: address = Web3.to_checksum_address(member["address"]) joined_at = member.get("joined_at") shares = member.get("shares", "0") loot = member.get("loot", "0") # Try to resolve ENS name ens_name = self.ens_resolver.get_ens_name(address) # Check if contact already exists query = 'SELECT id FROM "Contact" WHERE "ethereumAddress" = %(address)s' result = self.db.execute_query(query, {"address": address}) if result: # Contact exists, update it contact_id = result[0]["id"] if ens_name: self.db.update_contact(contact_id, {"ensName": ens_name}) members_updated += 1 else: # Contact doesn't exist, create it contact_id = self.db.upsert_contact( ethereum_address=address, ens_name=ens_name ) members_added += 1 # Add DAO membership self.db.add_dao_membership( contact_id=contact_id, dao_name=self.dao_name, dao_type="Moloch", joined_at=joined_at ) # Add a tag for the DAO self.db.add_tag_to_contact( contact_id=contact_id, tag_name=self.dao_name, color="#FF5733" # Example color ) # Add a note with additional information note_content = f"{self.dao_name} Membership Information:\n" note_content += f"Shares: {shares}\n" note_content += f"Loot: {loot}\n" if joined_at: note_content += f"Joined: {joined_at}\n" self.db.add_note_to_contact(contact_id, note_content) # If we have an ENS name, try to get additional profile information if ens_name: self.ens_resolver.update_contact_from_ens(contact_id, ens_name) # Rate limiting to avoid API throttling time.sleep(0.1) logger.info(f"Added {members_added} new contacts and updated {members_updated} existing contacts") def run(self) -> None: """Run the scraper to fetch and process DAO members.""" members = self.get_dao_members() if members: self.process_members(members) logger.info("DAO members scraping completed successfully") else: logger.warning("No members found or error occurred") def main(): """Main entry point for the script.""" parser = argparse.ArgumentParser(description="Scrape Moloch DAO members") parser.add_argument("--dao-address", required=True, help="DAO contract address") parser.add_argument("--dao-name", required=True, help="DAO name") parser.add_argument("--network", default="0x1", help="Network ID (0x1 for Ethereum, 0x64 for Gnosis Chain)") args = parser.parse_args() scraper = MolochDAOScraper(args.dao_address, args.dao_name, args.network) scraper.run() if __name__ == "__main__": main()