#!/usr/bin/env python3 """ Import Public Haus Members by Querying Token Holders This script fetches members of Public Haus DAO by querying holders of both the voting (shares) and non-voting (loot) tokens, imports them into the database, and links them to the Public Haus DAO. Usage: python import_public_haus_tokens.py """ import os import sys import logging import json import time import requests from typing import Dict, Any, List, Optional, Set from web3 import Web3 from dotenv import load_dotenv # Add parent directory to path to import utils sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from utils.db_connector import DatabaseConnector from utils.logger import setup_logger from utils.ens_resolver import ENSResolver # Load environment variables load_dotenv() # Setup logging logger = setup_logger("public_haus_tokens_importer") # Constants PUBLIC_HAUS_DAO_ID = "0xf5d6b637a9185707f52d40d452956ca49018247a" # Public Haus DAO ID on Optimism SHARES_TOKEN_ADDRESS = "0x4950c436F69c8b4F80f688edc814C5bA84Aa70f5" # Voting token (shares) LOOT_TOKEN_ADDRESS = "0xab6033E3EC2144FB279fe68dA92B7aC6a42Da6d8" # Non-voting token (loot) # Optimism Etherscan API OPTIMISM_ETHERSCAN_API_URL = "https://api-optimistic.etherscan.io/api" # ERC20 ABI (minimal for balance checking) ERC20_ABI = [ { "constant": True, "inputs": [{"name": "_owner", "type": "address"}], "name": "balanceOf", "outputs": [{"name": "balance", "type": "uint256"}], "type": "function" }, { "constant": True, "inputs": [], "name": "totalSupply", "outputs": [{"name": "", "type": "uint256"}], "type": "function" }, { "constant": True, "inputs": [], "name": "name", "outputs": [{"name": "", "type": "string"}], "type": "function" }, { "constant": True, "inputs": [], "name": "symbol", "outputs": [{"name": "", "type": "string"}], "type": "function" }, { "constant": True, "inputs": [], "name": "decimals", "outputs": [{"name": "", "type": "uint8"}], "type": "function" } ] class PublicHausTokensImporter: """Importer for Public Haus DAO members based on token holdings""" def __init__(self): """Initialize the importer""" # Initialize database self.db = DatabaseConnector() # Initialize Web3 optimism_rpc_url = os.getenv("OPTIMISM_RPC_URL") if not optimism_rpc_url: raise ValueError("OPTIMISM_RPC_URL environment variable not set") self.web3 = Web3(Web3.HTTPProvider(optimism_rpc_url)) if not self.web3.is_connected(): raise ValueError("Failed to connect to Optimism RPC") logger.info(f"Connected to Optimism: {self.web3.is_connected()}") # Initialize token contracts self.shares_token = self.web3.eth.contract( address=self.web3.to_checksum_address(SHARES_TOKEN_ADDRESS), abi=ERC20_ABI ) self.loot_token = self.web3.eth.contract( address=self.web3.to_checksum_address(LOOT_TOKEN_ADDRESS), abi=ERC20_ABI ) # Get Etherscan API key self.etherscan_api_key = os.getenv("OPTIMISM_ETHERSCAN_API_KEY") if not self.etherscan_api_key: logger.warning("OPTIMISM_ETHERSCAN_API_KEY not set, using API without key (rate limited)") self.etherscan_api_key = "" else: logger.info("Using Optimism Etherscan API key") # Initialize ENS resolver for Ethereum mainnet ethereum_rpc_url = os.getenv("ETHEREUM_RPC_URL", "https://eth-mainnet.g.alchemy.com/v2/1fkIfqUX_MoHhd4Qqnu9UItM8Fc4Ls2q") self.eth_web3 = Web3(Web3.HTTPProvider(ethereum_rpc_url)) if not self.eth_web3.is_connected(): logger.warning("Failed to connect to Ethereum RPC for ENS resolution") self.ens_resolver = None else: logger.info(f"Connected to Ethereum for ENS resolution: {self.eth_web3.is_connected()}") self.ens_resolver = ENSResolver(self.eth_web3) # Register data source self.data_source_id = self.register_data_source() # Initialize scraping job self.job_id = self.db.create_scraping_job( source_name="Public Haus DAO Tokens", status="running" ) logger.info(f"Created scraping job with ID: {self.job_id}") def register_data_source(self) -> str: """Register the Public Haus data source in the database""" return self.db.upsert_data_source( name="Public Haus DAO Tokens", source_type="blockchain", description="Public Haus DAO members identified by token holdings" ) def get_token_info(self, token_contract, token_name) -> Dict[str, Any]: """ Get information about a token Args: token_contract: Web3 contract instance token_name: Name of the token for logging Returns: Token information """ try: name = token_contract.functions.name().call() symbol = token_contract.functions.symbol().call() decimals = token_contract.functions.decimals().call() total_supply = token_contract.functions.totalSupply().call() token_info = { "name": name, "symbol": symbol, "decimals": decimals, "totalSupply": total_supply } logger.info(f"{token_name} info: {name} ({symbol})") logger.info(f"{token_name} total supply: {total_supply / (10 ** decimals):.2f} {symbol}") return token_info except Exception as e: logger.error(f"Error getting {token_name} info via Web3: {e}") # Try Etherscan API as fallback try: token_address = token_contract.address params = { "module": "token", "action": "tokeninfo", "contractaddress": token_address, "apikey": self.etherscan_api_key } response = requests.get(OPTIMISM_ETHERSCAN_API_URL, params=params) data = response.json() if data["status"] == "1": token_info = data["result"][0] logger.info(f"{token_name} info from Etherscan: {token_info.get('name')} ({token_info.get('symbol')})") return { "name": token_info.get("name", f"Public Haus {token_name}"), "symbol": token_info.get("symbol", token_name.upper()), "decimals": int(token_info.get("decimals", 18)), "totalSupply": int(token_info.get("totalSupply", "0")) } except Exception as etherscan_error: logger.error(f"Error getting {token_name} info via Etherscan: {etherscan_error}") # Return default values if both methods fail return { "name": f"Public Haus {token_name}", "symbol": token_name.upper(), "decimals": 18, "totalSupply": 0 } def fetch_token_holders_via_etherscan(self, token_address, token_name, decimals) -> List[Dict[str, Any]]: """ Fetch holders of a token using Etherscan API Args: token_address: Address of the token token_name: Name of the token for logging decimals: Token decimals Returns: List of token holders with their balances """ try: # Get token holders from Etherscan params = { "module": "token", "action": "tokenholderlist", "contractaddress": token_address, "page": 1, "offset": 100, # Get up to 100 holders "apikey": self.etherscan_api_key } response = requests.get(OPTIMISM_ETHERSCAN_API_URL, params=params) data = response.json() holders = [] if data["status"] == "1": for holder in data["result"]: address = holder["address"] balance = int(holder["TokenHolderQuantity"]) # Skip zero balances if balance > 0: holders.append({ "address": address, "balance": balance, "balanceFormatted": balance / (10 ** decimals), "tokenType": token_name }) logger.info(f"Found {len(holders)} {token_name} holders with non-zero balance via Etherscan") return holders else: logger.warning(f"Error getting {token_name} holders from Etherscan: {data.get('message')}") return [] except Exception as e: logger.error(f"Error fetching {token_name} holders via Etherscan: {e}") return [] def fetch_token_transfers_via_etherscan(self, token_address, token_name, decimals) -> List[Dict[str, Any]]: """ Fetch token transfers using Etherscan API and extract unique addresses Args: token_address: Address of the token token_name: Name of the token for logging decimals: Token decimals Returns: List of token holders with their balances """ try: # Get token transfers from Etherscan params = { "module": "account", "action": "tokentx", "contractaddress": token_address, "page": 1, "offset": 1000, # Get up to 1000 transfers "sort": "desc", "apikey": self.etherscan_api_key } response = requests.get(OPTIMISM_ETHERSCAN_API_URL, params=params) data = response.json() addresses = set() if data["status"] == "1": for tx in data["result"]: addresses.add(tx["to"]) addresses.add(tx["from"]) # Remove zero address if "0x0000000000000000000000000000000000000000" in addresses: addresses.remove("0x0000000000000000000000000000000000000000") # Create holder objects holders = [] for address in addresses: holders.append({ "address": address, "balance": 1, # We don't know the actual balance "balanceFormatted": 1, "tokenType": token_name }) logger.info(f"Found {len(holders)} unique addresses from {token_name} transfers via Etherscan") return holders else: logger.warning(f"Error getting {token_name} transfers from Etherscan: {data.get('message')}") return [] except Exception as e: logger.error(f"Error fetching {token_name} transfers via Etherscan: {e}") return [] def fetch_token_holders_via_web3(self, token_contract, token_name, decimals) -> List[Dict[str, Any]]: """ Fetch holders of a token by checking balances of known addresses Args: token_contract: Web3 contract instance token_name: Name of the token for logging decimals: Token decimals Returns: List of token holders with their balances """ try: # Get the latest block number latest_block = self.web3.eth.block_number # Try to get some known addresses from recent transactions to the token contract known_addresses = set() # Check the last 100 blocks for transactions to the token contract for block_num in range(max(0, latest_block - 100), latest_block + 1): try: block = self.web3.eth.get_block(block_num, full_transactions=True) for tx in block.transactions: if tx.to and tx.to.lower() == token_contract.address.lower(): known_addresses.add(tx['from']) except Exception as e: logger.warning(f"Error getting block {block_num}: {e}") continue # Add the DAO address as a known address known_addresses.add(self.web3.to_checksum_address(PUBLIC_HAUS_DAO_ID)) # Check balances for known addresses holders = [] for address in known_addresses: try: balance = token_contract.functions.balanceOf(address).call() # Only include addresses with non-zero balance if balance > 0: holders.append({ "address": address, "balance": balance, "balanceFormatted": balance / (10 ** decimals), "tokenType": token_name }) except Exception as e: logger.error(f"Error checking {token_name} balance for {address}: {e}") logger.info(f"Found {len(holders)} {token_name} holders with non-zero balance via Web3") return holders except Exception as e: logger.error(f"Error fetching {token_name} holders via Web3: {e}") return [] def fetch_all_token_holders(self) -> List[Dict[str, Any]]: """ Fetch holders of both shares and loot tokens Returns: List of token holders with their balances """ all_holders = [] # Get token info shares_info = self.get_token_info(self.shares_token, "Shares") loot_info = self.get_token_info(self.loot_token, "Loot") shares_decimals = shares_info["decimals"] loot_decimals = loot_info["decimals"] # Try different methods to get token holders # 1. Try Etherscan tokenholderlist endpoint shares_holders = self.fetch_token_holders_via_etherscan( SHARES_TOKEN_ADDRESS, "Shares", shares_decimals ) loot_holders = self.fetch_token_holders_via_etherscan( LOOT_TOKEN_ADDRESS, "Loot", loot_decimals ) # 2. If that fails, try getting transfers if not shares_holders: shares_holders = self.fetch_token_transfers_via_etherscan( SHARES_TOKEN_ADDRESS, "Shares", shares_decimals ) if not loot_holders: loot_holders = self.fetch_token_transfers_via_etherscan( LOOT_TOKEN_ADDRESS, "Loot", loot_decimals ) # 3. If that fails, try Web3 if not shares_holders: shares_holders = self.fetch_token_holders_via_web3( self.shares_token, "Shares", shares_decimals ) if not loot_holders: loot_holders = self.fetch_token_holders_via_web3( self.loot_token, "Loot", loot_decimals ) # Combine holders all_holders.extend(shares_holders) all_holders.extend(loot_holders) # If we still don't have any holders, use the DAO address itself if not all_holders: logger.warning("No token holders found, using DAO address as fallback") all_holders.append({ "address": PUBLIC_HAUS_DAO_ID, "balance": 1, "balanceFormatted": 1, "tokenType": "Fallback" }) return all_holders def merge_holders(self, holders: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ Merge holders by address, combining shares and loot Args: holders: List of token holders Returns: List of merged holders """ merged = {} for holder in holders: address = holder["address"] balance = holder["balance"] token_type = holder["tokenType"] if address not in merged: merged[address] = { "address": address, "shares": 0, "sharesFormatted": 0, "loot": 0, "lootFormatted": 0, "dao": "Public Haus" } if token_type == "Shares": merged[address]["shares"] = balance merged[address]["sharesFormatted"] = holder["balanceFormatted"] elif token_type == "Loot": merged[address]["loot"] = balance merged[address]["lootFormatted"] = holder["balanceFormatted"] else: # Fallback merged[address]["shares"] = balance merged[address]["sharesFormatted"] = holder["balanceFormatted"] return list(merged.values()) def resolve_ens_name(self, address: str) -> Optional[str]: """ Resolve ENS name for an Ethereum address Args: address: Ethereum address to resolve Returns: ENS name if found, None otherwise """ if not self.ens_resolver: return None try: ens_name = self.ens_resolver.get_ens_name(address) if ens_name: logger.info(f"Resolved ENS name for {address}: {ens_name}") return ens_name except Exception as e: logger.error(f"Error resolving ENS name for {address}: {e}") return None def process_holder(self, holder: Dict[str, Any]) -> Optional[str]: """ Process a token holder and import into the database Args: holder: Token holder information Returns: Contact ID if successful, None otherwise """ try: # Extract holder information address = holder["address"] shares = holder["shares"] shares_formatted = holder["sharesFormatted"] loot = holder["loot"] loot_formatted = holder["lootFormatted"] dao_name = holder["dao"] # Check if contact exists query = 'SELECT id, name, "ensName" FROM "Contact" WHERE "ethereumAddress" ILIKE %(address)s' existing_contacts = self.db.execute_query(query, {"address": address}) contact_id = None if existing_contacts: # Use existing contact contact_id = existing_contacts[0]["id"] logger.info(f"Found existing contact {contact_id} for address {address}") else: # Create new contact contact_id = self.db.upsert_contact( ethereum_address=address, ens_name=None ) logger.info(f"Created new contact {contact_id} for address {address}") # Add DAO membership self.db.execute_update( """ INSERT INTO "DaoMembership" (id, "contactId", "daoName", "daoType", "createdAt", "updatedAt") VALUES (gen_random_uuid(), %(contact_id)s, %(dao_name)s, %(dao_type)s, NOW(), NOW()) ON CONFLICT ("contactId", "daoName") DO UPDATE SET "updatedAt" = NOW() """, { "contact_id": contact_id, "dao_name": dao_name, "dao_type": "Moloch V3" } ) # Add note about membership with token holdings note_content = f"Public Haus DAO Member\nShares: {shares_formatted}\nLoot: {loot_formatted}" self.db.add_note_to_contact( contact_id=contact_id, content=note_content ) # Add tags for the DAO and token holdings self.db.add_tag_to_contact( contact_id=contact_id, tag_name=dao_name ) if shares > 0: self.db.add_tag_to_contact( contact_id=contact_id, tag_name=f"{dao_name} Voting Member" ) if loot > 0: self.db.add_tag_to_contact( contact_id=contact_id, tag_name=f"{dao_name} Non-Voting Member" ) # Link to data source self.db.link_contact_to_data_source(contact_id, self.data_source_id) return contact_id except Exception as e: logger.error(f"Error processing holder {holder.get('address')}: {e}") return None def process_address(self, address: str, shares_balance: float = 0, loot_balance: float = 0) -> Optional[str]: """ Process a single address, creating or updating a contact and linking to the DAO Args: address: Ethereum address shares_balance: Balance of shares token loot_balance: Balance of loot token Returns: Contact ID if successful, None otherwise """ try: # Check if contact already exists query = 'SELECT id, name, "ensName" FROM "Contact" WHERE "ethereumAddress" ILIKE %(address)s' result = self.db.execute_query(query, {"address": address}) contact_id = None is_new = False # Resolve ENS name if needed ens_name = None if not result or not result[0].get("ensName"): ens_name = self.resolve_ens_name(address) if result: # Contact exists, get ID contact_id = result[0]["id"] logger.info(f"Found existing contact for {address}: {contact_id}") # Update ENS name if we found one and it's not already set if ens_name and not result[0].get("ensName"): self.db.update_contact(contact_id, {"ensName": ens_name}) logger.info(f"Updated ENS name for contact {contact_id}: {ens_name}") else: # Create new contact # Use ENS name as contact name if available, otherwise use address contact_name = ens_name.split('.')[0] if ens_name else f"ETH_{address[:8]}" contact_id = self.db.upsert_contact( ethereum_address=address, ens_name=ens_name ) logger.info(f"Created new contact for {address}: {contact_id}") is_new = True # Add DAO membership self.db.add_dao_membership( contact_id=contact_id, dao_name="Public Haus", dao_type="Moloch V3" ) # Add note about membership with token holdings note_content = f"Public Haus DAO Member\nShares: {shares_balance}\nLoot: {loot_balance}" self.db.add_note_to_contact( contact_id=contact_id, content=note_content ) # Add tags for the DAO and token holdings self.db.add_tag_to_contact( contact_id=contact_id, tag_name="Public Haus" ) if shares_balance > 0: self.db.add_tag_to_contact( contact_id=contact_id, tag_name="Public Haus Voting Member" ) if loot_balance > 0: self.db.add_tag_to_contact( contact_id=contact_id, tag_name="Public Haus Non-Voting Member" ) # Link to data source self.db.link_contact_to_data_source(contact_id, self.data_source_id) return contact_id except Exception as e: logger.error(f"Error processing address {address}: {e}") return None def run(self) -> int: """ Run the importer Returns: Number of holders imported """ try: # Fetch token holders all_holders = self.fetch_all_token_holders() # Merge holders by address merged_holders = self.merge_holders(all_holders) if not merged_holders: logger.info("No token holders found") self.db.update_scraping_job(self.job_id, "completed") return 0 # Process holders imported_count = 0 existing_count = 0 for holder in merged_holders: try: contact_id = self.process_address( address=holder["address"], shares_balance=holder["shares"], loot_balance=holder["loot"] ) if contact_id: imported_count += 1 except Exception as e: logger.exception(f"Error processing holder {holder.get('address')}: {e}") # Add a small delay to avoid overwhelming the database time.sleep(0.1) # Complete the scraping job self.db.update_scraping_job( self.job_id, "completed", records_processed=len(merged_holders), records_added=imported_count, records_updated=existing_count ) logger.info(f"Imported {imported_count} holders out of {len(merged_holders)} processed") # Run ENS resolution for any contacts that don't have ENS names if self.ens_resolver: logger.info("Running ENS resolution for contacts without ENS names...") from utils.resolve_ens_names import ENSResolver as BatchENSResolver batch_resolver = BatchENSResolver() batch_resolver.run(batch_size=50, delay_seconds=0.5) return imported_count except Exception as e: # Update the scraping job with error self.db.update_scraping_job(self.job_id, "failed", error_message=str(e)) logger.exception(f"Error importing holders: {e}") raise def main(): """Main function""" try: importer = PublicHausTokensImporter() imported_count = importer.run() logger.info(f"Import completed successfully. Imported {imported_count} token holders.") return 0 except Exception as e: logger.exception(f"Error importing token holders: {e}") return 1 if __name__ == "__main__": sys.exit(main())