310 lines
11 KiB
Python
Executable File
310 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Moloch DAO Scraper
|
|
|
|
This script fetches all members of a specific Moloch DAO and stores their
|
|
Ethereum addresses in the database. It also attempts to resolve ENS names
|
|
for the addresses.
|
|
|
|
Usage:
|
|
python moloch_dao_scraper.py --dao-address 0xfe1084bc16427e5eb7f13fc19bcd4e641f7d571f --dao-name "Raid Guild" --network 0x64
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import argparse
|
|
import json
|
|
import time
|
|
from datetime import datetime
|
|
from typing import Dict, List, Optional, Any
|
|
import requests
|
|
from web3 import Web3
|
|
from dotenv import load_dotenv
|
|
|
|
# Add parent directory to path to import utils
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from utils.db_connector import DatabaseConnector
|
|
from utils.ens_resolver import ENSResolver
|
|
from utils.logger import setup_logger
|
|
|
|
# Load environment variables
|
|
load_dotenv()
|
|
|
|
# Setup logging
|
|
logger = setup_logger("moloch_dao_scraper")
|
|
|
|
# Moloch DAO ABI (partial, only the functions we need)
|
|
MOLOCH_DAO_ABI = [
|
|
{
|
|
"constant": True,
|
|
"inputs": [],
|
|
"name": "memberCount",
|
|
"outputs": [{"name": "", "type": "uint256"}],
|
|
"payable": False,
|
|
"stateMutability": "view",
|
|
"type": "function"
|
|
},
|
|
{
|
|
"constant": True,
|
|
"inputs": [{"name": "", "type": "address"}],
|
|
"name": "members",
|
|
"outputs": [
|
|
{"name": "delegateKey", "type": "address"},
|
|
{"name": "shares", "type": "uint256"},
|
|
{"name": "loot", "type": "uint256"},
|
|
{"name": "exists", "type": "bool"},
|
|
{"name": "highestIndexYesVote", "type": "uint256"},
|
|
{"name": "jailed", "type": "uint256"}
|
|
],
|
|
"payable": False,
|
|
"stateMutability": "view",
|
|
"type": "function"
|
|
},
|
|
{
|
|
"constant": True,
|
|
"inputs": [{"name": "", "type": "uint256"}],
|
|
"name": "memberAddressByIndex",
|
|
"outputs": [{"name": "", "type": "address"}],
|
|
"payable": False,
|
|
"stateMutability": "view",
|
|
"type": "function"
|
|
}
|
|
]
|
|
|
|
class MolochDAOScraper:
|
|
"""Scraper for Moloch DAO members."""
|
|
|
|
def __init__(self, dao_address: str, dao_name: str, network: str = "0x1"):
|
|
"""
|
|
Initialize the Moloch DAO scraper.
|
|
|
|
Args:
|
|
dao_address: Ethereum address of the DAO contract
|
|
dao_name: Name of the DAO
|
|
network: Network ID (default: "0x1" for Ethereum mainnet, "0x64" for Gnosis Chain)
|
|
"""
|
|
self.dao_address = Web3.to_checksum_address(dao_address)
|
|
self.dao_name = dao_name
|
|
self.network = network
|
|
self.alchemy_api_key = os.getenv("ALCHEMY_API_KEY")
|
|
|
|
# Set up Web3 provider based on network
|
|
if network == "0x1":
|
|
# Ethereum mainnet
|
|
provider_url = f"https://eth-mainnet.g.alchemy.com/v2/{self.alchemy_api_key}"
|
|
elif network == "0x64":
|
|
# Gnosis Chain (xDai)
|
|
provider_url = "https://rpc.gnosischain.com"
|
|
else:
|
|
logger.error(f"Unsupported network: {network}")
|
|
sys.exit(1)
|
|
|
|
self.web3 = Web3(Web3.HTTPProvider(provider_url))
|
|
self.db = DatabaseConnector()
|
|
self.ens_resolver = ENSResolver(self.web3)
|
|
|
|
# Initialize the DAO contract
|
|
self.dao_contract = self.web3.eth.contract(
|
|
address=self.dao_address,
|
|
abi=MOLOCH_DAO_ABI
|
|
)
|
|
|
|
# Validate API keys
|
|
if not self.alchemy_api_key:
|
|
logger.error("ALCHEMY_API_KEY not found in environment variables")
|
|
sys.exit(1)
|
|
|
|
# Register data source
|
|
self.register_data_source()
|
|
|
|
def register_data_source(self) -> None:
|
|
"""Register this DAO as a data source in the database."""
|
|
self.db.upsert_data_source(
|
|
name=f"DAO:{self.dao_name}",
|
|
source_type="DAO",
|
|
description=f"Members of {self.dao_name} DAO ({self.dao_address})"
|
|
)
|
|
|
|
def get_dao_members(self) -> List[Dict[str, Any]]:
|
|
"""
|
|
Fetch all members of the DAO by directly querying the contract.
|
|
|
|
Returns:
|
|
List of dictionaries containing member addresses and shares/loot
|
|
"""
|
|
logger.info(f"Fetching members for {self.dao_name} ({self.dao_address})")
|
|
|
|
# Start a scraping job
|
|
job_id = self.db.create_scraping_job(
|
|
source_name=f"DAO:{self.dao_name}",
|
|
status="running"
|
|
)
|
|
|
|
members = []
|
|
try:
|
|
# Get the total number of members
|
|
try:
|
|
member_count = self.dao_contract.functions.memberCount().call()
|
|
logger.info(f"Member count from contract: {member_count}")
|
|
except Exception as e:
|
|
logger.warning(f"Could not get member count: {str(e)}")
|
|
# If memberCount function is not available, we'll try a different approach
|
|
member_count = 0
|
|
index = 0
|
|
while True:
|
|
try:
|
|
# Try to get member address at index
|
|
address = self.dao_contract.functions.memberAddressByIndex(index).call()
|
|
if address != "0x0000000000000000000000000000000000000000":
|
|
member_count += 1
|
|
index += 1
|
|
else:
|
|
break
|
|
except Exception:
|
|
# If we get an error, we've reached the end of the list
|
|
break
|
|
logger.info(f"Estimated member count: {member_count}")
|
|
|
|
# Fetch all member addresses
|
|
member_addresses = []
|
|
for i in range(member_count):
|
|
try:
|
|
address = self.dao_contract.functions.memberAddressByIndex(i).call()
|
|
if address != "0x0000000000000000000000000000000000000000":
|
|
member_addresses.append(address)
|
|
except Exception as e:
|
|
logger.warning(f"Error getting member at index {i}: {str(e)}")
|
|
continue
|
|
|
|
logger.info(f"Found {len(member_addresses)} member addresses")
|
|
|
|
# Get member details for each address
|
|
for address in member_addresses:
|
|
try:
|
|
member_data = self.dao_contract.functions.members(address).call()
|
|
|
|
# Check if the member exists
|
|
if not member_data[3]: # exists flag
|
|
continue
|
|
|
|
members.append({
|
|
"address": address,
|
|
"shares": str(member_data[1]), # shares
|
|
"loot": str(member_data[2]), # loot
|
|
"joined_at": None # We don't have this information from the contract
|
|
})
|
|
except Exception as e:
|
|
logger.warning(f"Error getting member data for {address}: {str(e)}")
|
|
continue
|
|
|
|
# Update job with success
|
|
self.db.update_scraping_job(
|
|
job_id=job_id,
|
|
status="completed",
|
|
records_processed=len(member_addresses),
|
|
records_added=len(members)
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching DAO members: {str(e)}")
|
|
self.db.update_scraping_job(job_id, "failed", error_message=str(e))
|
|
return []
|
|
|
|
logger.info(f"Found {len(members)} DAO members")
|
|
return members
|
|
|
|
def process_members(self, members: List[Dict[str, Any]]) -> None:
|
|
"""
|
|
Process the list of members and store in database.
|
|
|
|
Args:
|
|
members: List of dictionaries containing member addresses and shares/loot
|
|
"""
|
|
logger.info(f"Processing {len(members)} members")
|
|
|
|
members_added = 0
|
|
members_updated = 0
|
|
|
|
for member in members:
|
|
address = Web3.to_checksum_address(member["address"])
|
|
joined_at = member.get("joined_at")
|
|
shares = member.get("shares", "0")
|
|
loot = member.get("loot", "0")
|
|
|
|
# Try to resolve ENS name
|
|
ens_name = self.ens_resolver.get_ens_name(address)
|
|
|
|
# Check if contact already exists
|
|
query = 'SELECT id FROM "Contact" WHERE "ethereumAddress" = %(address)s'
|
|
result = self.db.execute_query(query, {"address": address})
|
|
|
|
if result:
|
|
# Contact exists, update it
|
|
contact_id = result[0]["id"]
|
|
if ens_name:
|
|
self.db.update_contact(contact_id, {"ensName": ens_name})
|
|
members_updated += 1
|
|
else:
|
|
# Contact doesn't exist, create it
|
|
contact_id = self.db.upsert_contact(
|
|
ethereum_address=address,
|
|
ens_name=ens_name
|
|
)
|
|
members_added += 1
|
|
|
|
# Add DAO membership
|
|
self.db.add_dao_membership(
|
|
contact_id=contact_id,
|
|
dao_name=self.dao_name,
|
|
dao_type="Moloch",
|
|
joined_at=joined_at
|
|
)
|
|
|
|
# Add a tag for the DAO
|
|
self.db.add_tag_to_contact(
|
|
contact_id=contact_id,
|
|
tag_name=self.dao_name,
|
|
color="#FF5733" # Example color
|
|
)
|
|
|
|
# Add a note with additional information
|
|
note_content = f"{self.dao_name} Membership Information:\n"
|
|
note_content += f"Shares: {shares}\n"
|
|
note_content += f"Loot: {loot}\n"
|
|
if joined_at:
|
|
note_content += f"Joined: {joined_at}\n"
|
|
|
|
self.db.add_note_to_contact(contact_id, note_content)
|
|
|
|
# If we have an ENS name, try to get additional profile information
|
|
if ens_name:
|
|
self.ens_resolver.update_contact_from_ens(contact_id, ens_name)
|
|
|
|
# Rate limiting to avoid API throttling
|
|
time.sleep(0.1)
|
|
|
|
logger.info(f"Added {members_added} new contacts and updated {members_updated} existing contacts")
|
|
|
|
def run(self) -> None:
|
|
"""Run the scraper to fetch and process DAO members."""
|
|
members = self.get_dao_members()
|
|
if members:
|
|
self.process_members(members)
|
|
logger.info("DAO members scraping completed successfully")
|
|
else:
|
|
logger.warning("No members found or error occurred")
|
|
|
|
def main():
|
|
"""Main entry point for the script."""
|
|
parser = argparse.ArgumentParser(description="Scrape Moloch DAO members")
|
|
parser.add_argument("--dao-address", required=True, help="DAO contract address")
|
|
parser.add_argument("--dao-name", required=True, help="DAO name")
|
|
parser.add_argument("--network", default="0x1", help="Network ID (0x1 for Ethereum, 0x64 for Gnosis Chain)")
|
|
|
|
args = parser.parse_args()
|
|
|
|
scraper = MolochDAOScraper(args.dao_address, args.dao_name, args.network)
|
|
scraper.run()
|
|
|
|
if __name__ == "__main__":
|
|
main() |