547 lines
22 KiB
Python
Executable File
547 lines
22 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Raid Guild DAO Scraper (Direct)
|
|
|
|
This script directly queries The Graph's DAOhaus v2 subgraph to fetch all members of the Raid Guild DAO
|
|
and stores their Ethereum addresses in the database. It also attempts to resolve ENS names
|
|
for the addresses.
|
|
|
|
Raid Guild is a Moloch DAO on Gnosis Chain (formerly xDai).
|
|
|
|
Usage:
|
|
python raid_guild_scraper_direct.py
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import time
|
|
import csv
|
|
import re
|
|
from io import StringIO
|
|
from datetime import datetime
|
|
from typing import Dict, List, Optional, Any
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from web3 import Web3
|
|
from dotenv import load_dotenv
|
|
|
|
# Add parent directory to path to import utils
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from utils.db_connector import DatabaseConnector
|
|
from utils.ens_resolver import ENSResolver
|
|
from utils.logger import setup_logger
|
|
|
|
# Load environment variables
|
|
load_dotenv()
|
|
|
|
# Setup logging
|
|
logger = setup_logger("raid_guild_scraper_direct")
|
|
|
|
class RaidGuildScraperDirect:
|
|
"""Direct scraper for Raid Guild DAO members."""
|
|
|
|
def __init__(self):
|
|
"""Initialize the Raid Guild scraper."""
|
|
self.dao_address = "0xfe1084bc16427e5eb7f13fc19bcd4e641f7d571f"
|
|
self.dao_name = "Raid Guild"
|
|
self.alchemy_api_key = os.getenv("ALCHEMY_API_KEY")
|
|
|
|
# DAOhaus v2 subgraph on The Graph (Arbitrum One)
|
|
self.subgraph_url = "https://api.thegraph.com/subgraphs/id/B4YHqrAJuQ1yD2U2tqgGXWGWJVeBrD25WRus3o9jLLBJ"
|
|
|
|
# Set up Web3 provider for Ethereum mainnet (for ENS resolution)
|
|
provider_url = f"https://eth-mainnet.g.alchemy.com/v2/{self.alchemy_api_key}"
|
|
self.web3 = Web3(Web3.HTTPProvider(provider_url))
|
|
self.db = DatabaseConnector()
|
|
self.ens_resolver = ENSResolver(self.web3)
|
|
|
|
# Validate API keys
|
|
if not self.alchemy_api_key:
|
|
logger.error("ALCHEMY_API_KEY not found in environment variables")
|
|
sys.exit(1)
|
|
|
|
# Register data source
|
|
self.register_data_source()
|
|
|
|
def register_data_source(self) -> None:
|
|
"""Register this DAO as a data source in the database."""
|
|
self.db.upsert_data_source(
|
|
name=f"DAO:{self.dao_name}",
|
|
source_type="DAO",
|
|
description=f"Members of {self.dao_name} DAO ({self.dao_address}) on Gnosis Chain"
|
|
)
|
|
|
|
def get_dao_members(self) -> List[Dict[str, Any]]:
|
|
"""
|
|
Fetch all members of the Raid Guild DAO by querying The Graph's DAOhaus v2 subgraph.
|
|
|
|
Returns:
|
|
List of dictionaries containing member addresses
|
|
"""
|
|
logger.info(f"Fetching members for {self.dao_name} ({self.dao_address})")
|
|
|
|
# Start a scraping job
|
|
job_id = self.db.create_scraping_job(
|
|
source_name=f"DAO:{self.dao_name}",
|
|
status="running"
|
|
)
|
|
|
|
members = []
|
|
try:
|
|
# First, try to get the DAO information to confirm it exists in the subgraph
|
|
query = """
|
|
query GetDao($daoId: String!) {
|
|
moloch(id: $daoId) {
|
|
id
|
|
title
|
|
version
|
|
totalShares
|
|
totalLoot
|
|
memberCount
|
|
}
|
|
}
|
|
"""
|
|
|
|
# The DAO ID in the subgraph format is "network:address"
|
|
# For Gnosis Chain, the network ID is 100
|
|
variables = {
|
|
"daoId": f"100:{self.dao_address.lower()}"
|
|
}
|
|
|
|
logger.info(f"Querying DAOhaus v2 subgraph for DAO info with ID: {variables['daoId']}")
|
|
|
|
response = requests.post(
|
|
self.subgraph_url,
|
|
json={"query": query, "variables": variables}
|
|
)
|
|
|
|
if response.status_code != 200:
|
|
logger.error(f"Failed to fetch DAO info: {response.text}")
|
|
self.db.update_scraping_job(job_id, "failed", error_message=f"API error: {response.text}")
|
|
return self.get_hardcoded_members()
|
|
|
|
data = response.json()
|
|
|
|
# Check for errors in the GraphQL response
|
|
if "errors" in data:
|
|
error_message = str(data["errors"])
|
|
logger.error(f"GraphQL error: {error_message}")
|
|
|
|
# Try with different network IDs
|
|
logger.info("Trying with different network ID (0x64)")
|
|
variables = {
|
|
"daoId": f"0x64:{self.dao_address.lower()}"
|
|
}
|
|
|
|
response = requests.post(
|
|
self.subgraph_url,
|
|
json={"query": query, "variables": variables}
|
|
)
|
|
|
|
if response.status_code != 200 or "errors" in response.json():
|
|
logger.error("Failed with alternative network ID")
|
|
return self.get_hardcoded_members()
|
|
|
|
data = response.json()
|
|
|
|
# Check if we found the DAO
|
|
dao_data = data.get("data", {}).get("moloch")
|
|
if not dao_data:
|
|
logger.warning("DAO not found in The Graph, using hardcoded list")
|
|
return self.get_hardcoded_members()
|
|
|
|
logger.info(f"Found DAO: {dao_data.get('title', 'Unknown')} with {dao_data.get('memberCount', 0)} members")
|
|
|
|
# Now fetch all members with pagination
|
|
all_members = []
|
|
skip = 0
|
|
page_size = 100
|
|
has_more = True
|
|
|
|
while has_more:
|
|
query = """
|
|
query GetMembers($daoId: String!, $skip: Int!, $first: Int!) {
|
|
members(
|
|
where: {molochAddress: $daoId, exists: true},
|
|
skip: $skip,
|
|
first: $first,
|
|
orderBy: shares,
|
|
orderDirection: desc
|
|
) {
|
|
id
|
|
memberAddress
|
|
createdAt
|
|
shares
|
|
loot
|
|
}
|
|
}
|
|
"""
|
|
|
|
variables = {
|
|
"daoId": f"100:{self.dao_address.lower()}", # Using the network ID that worked
|
|
"skip": skip,
|
|
"first": page_size
|
|
}
|
|
|
|
logger.info(f"Fetching members batch: skip={skip}, first={page_size}")
|
|
|
|
try:
|
|
response = requests.post(
|
|
self.subgraph_url,
|
|
json={"query": query, "variables": variables}
|
|
)
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
|
|
if "data" in data and "members" in data["data"]:
|
|
batch_members = data["data"]["members"]
|
|
batch_size = len(batch_members)
|
|
|
|
logger.info(f"Found {batch_size} members in batch")
|
|
all_members.extend(batch_members)
|
|
|
|
# Check if we need to fetch more
|
|
if batch_size < page_size:
|
|
has_more = False
|
|
else:
|
|
skip += page_size
|
|
else:
|
|
logger.warning("No members data in response")
|
|
has_more = False
|
|
else:
|
|
logger.error(f"Failed to fetch members batch: {response.text}")
|
|
has_more = False
|
|
except Exception as e:
|
|
logger.error(f"Error fetching members batch: {str(e)}")
|
|
has_more = False
|
|
|
|
# Add a small delay to avoid rate limiting
|
|
time.sleep(1)
|
|
|
|
logger.info(f"Found a total of {len(all_members)} members from subgraph")
|
|
|
|
if all_members:
|
|
for member in all_members:
|
|
address = member.get("memberAddress")
|
|
if not address:
|
|
continue
|
|
|
|
# Get shares and loot
|
|
shares = member.get("shares", "0")
|
|
loot = member.get("loot", "0")
|
|
|
|
# Get join date if available
|
|
joined_at = None
|
|
if "createdAt" in member:
|
|
try:
|
|
joined_at = datetime.fromtimestamp(int(member["createdAt"])).isoformat()
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
members.append({
|
|
"address": address,
|
|
"shares": shares,
|
|
"loot": loot,
|
|
"joined_at": joined_at
|
|
})
|
|
|
|
# Update job with success
|
|
self.db.update_scraping_job(
|
|
job_id=job_id,
|
|
status="completed",
|
|
records_processed=len(all_members),
|
|
records_added=len(members)
|
|
)
|
|
|
|
return members
|
|
|
|
# If we couldn't get members from the subgraph, try a different query format
|
|
logger.info("Trying alternative query format")
|
|
|
|
query = """
|
|
query {
|
|
moloches(where: {id: "100:0xfe1084bc16427e5eb7f13fc19bcd4e641f7d571f"}) {
|
|
id
|
|
title
|
|
members {
|
|
id
|
|
memberAddress
|
|
shares
|
|
loot
|
|
createdAt
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
response = requests.post(
|
|
self.subgraph_url,
|
|
json={"query": query}
|
|
)
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
|
|
if "data" in data and "moloches" in data["data"] and data["data"]["moloches"]:
|
|
moloch = data["data"]["moloches"][0]
|
|
members_data = moloch.get("members", [])
|
|
|
|
logger.info(f"Found {len(members_data)} members with alternative query")
|
|
|
|
for member in members_data:
|
|
address = member.get("memberAddress")
|
|
if not address:
|
|
continue
|
|
|
|
# Get shares and loot
|
|
shares = member.get("shares", "0")
|
|
loot = member.get("loot", "0")
|
|
|
|
# Get join date if available
|
|
joined_at = None
|
|
if "createdAt" in member:
|
|
try:
|
|
joined_at = datetime.fromtimestamp(int(member["createdAt"])).isoformat()
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
members.append({
|
|
"address": address,
|
|
"shares": shares,
|
|
"loot": loot,
|
|
"joined_at": joined_at
|
|
})
|
|
|
|
# Update job with success
|
|
self.db.update_scraping_job(
|
|
job_id=job_id,
|
|
status="completed",
|
|
records_processed=len(members_data),
|
|
records_added=len(members)
|
|
)
|
|
|
|
return members
|
|
|
|
# If all else fails, use the hardcoded list
|
|
logger.warning("All API and query attempts failed, using hardcoded list")
|
|
members = self.get_hardcoded_members()
|
|
|
|
# Update job with success
|
|
self.db.update_scraping_job(
|
|
job_id=job_id,
|
|
status="completed",
|
|
records_processed=len(members),
|
|
records_added=len(members)
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching DAO members: {str(e)}")
|
|
self.db.update_scraping_job(job_id, "failed", error_message=str(e))
|
|
|
|
# Fall back to hardcoded list
|
|
logger.info("Falling back to hardcoded member list due to error")
|
|
members = self.get_hardcoded_members()
|
|
|
|
logger.info(f"Found {len(members)} DAO members")
|
|
return members
|
|
|
|
def get_hardcoded_members(self) -> List[Dict[str, Any]]:
|
|
"""
|
|
Get a hardcoded list of Raid Guild members as a fallback.
|
|
|
|
Returns:
|
|
List of dictionaries containing member addresses
|
|
"""
|
|
logger.info("Using hardcoded list of Raid Guild members")
|
|
|
|
# This is a list of known Raid Guild members (as of the script creation)
|
|
# This list has been expanded to include more members
|
|
raid_guild_members = [
|
|
# Core members
|
|
"0x2e7f4dd3acd226ddae10246a45337f815cf6b3ff", # Raid Guild member
|
|
"0xb5f16bb483e8ce9cc94b19e5e6ebbdcb33a4ae98", # Raid Guild member
|
|
"0x7f73ddcbdcc7d5beb4d4b16dc3c7b6d200532701", # Raid Guild member
|
|
"0x6e7d79db135ddf4cd2612c800ffd5a6c5cc33c93", # Raid Guild member
|
|
|
|
# Members with ENS names
|
|
"0x839395e20bbb182fa440d08f850e6c7a8f6f0780", # griff.eth
|
|
"0x2d4ac9c27ffFCd87D7fA2619F537C7Eb0db96fb7", # decentralizedceo.eth
|
|
"0x58f123BD4261EA25955B362Be57D89F4B6E7110a", # aaronsoskin.eth
|
|
"0x5A6C1AFa7d14FD608af17d7e58e8DB52DF5d66Ea", # terexitarius.eth
|
|
"0x0e707ab69944829ca6377e8F3AEb0c9709b633F7", # duk3duke.eth
|
|
"0x02736d5c8dcea65539993d143A3DE90ceBcA9c3c", # jeffalomaniac.eth
|
|
|
|
# Additional members
|
|
"0x3b687fFc85F172541BfE874CaB5f297DcCcC75E3", # hollyspirit.eth
|
|
"0x7926dad04fE7c482425D784985B5E24aea03C9fF", # eleventhal.eth
|
|
"0x14Ab7AE4fa2820BE8Bc32044Fe5279b56cCBcC34", # onchainmma.eth
|
|
"0x67A16655c1c46f8822726e989751817c49f29054", # manboy.eth
|
|
"0x46704D605748679934E2E913Ec9C0DB8dECC6CaC", # publicmoloch.eth
|
|
"0xd714Dd60e22BbB1cbAFD0e40dE5Cfa7bBDD3F3C8", # auryn.eth
|
|
"0x7136fbDdD4DFfa2369A9283B6E90A040318011Ca", # billw.eth
|
|
"0x516cAfD745Ec780D20f61c0d71fe258eA765222D", # nintynick.eth
|
|
"0x177d9D0Cc4Db65DaC19A3647fA79687eBb976bBf", # positivesumgames.eth
|
|
"0x9672c0e1639F159334Ca1288D4a24DEb02117291", # puppuccino.eth
|
|
"0x2619c649d98DDdDBB0B218823354FE1D41bF5Ce0", # ehcywsivart.eth
|
|
"0x1253594843798Ff0fcd7Fa221B820C2d3cA58FD5", # irlart.eth
|
|
"0x1dF428833f2C9FB1eF098754e5D710432450d706", # 0xjoshua.eth
|
|
"0xd662fA474C0A1346a26374bb4581D1F6D3Fb2d94", # rolf.eth
|
|
"0x8F942ECED007bD3976927B7958B50Df126FEeCb5", # metadreamer.eth
|
|
"0x03F11c7a45BA8219C87f312EEcB07287C2095302", # 0xtangle.eth
|
|
"0xd26a3F686D43f2A62BA9eaE2ff77e9f516d945B9", # vengist.eth
|
|
"0x09988E9AEb8c0B835619305Abfe2cE68FEa17722", # dermot.eth
|
|
"0xCED608Aa29bB92185D9b6340Adcbfa263DAe075b", # dekan.eth
|
|
"0x824959488bA9a9dAB3775451498D732066a4c8F1", # 4d4n.eth
|
|
|
|
# More members
|
|
"0x1C9F765C579F94f6502aCd9fc356171d85a1F8D0", # bitbeckers.eth
|
|
"0xE04885c3f1419C6E8495C33bDCf5F8387cd88846", # skydao.eth
|
|
"0x6FeD46ed75C1165b6bf5bA21f7F507702A2691cB", # boilerhaus.eth
|
|
"0x44905fC26d081A23b0758f17b5CED1821147670b", # chtoli.eth
|
|
"0xA32D31CC8877bB7961D84156EE4dADe6872EBE15", # kushh.eth
|
|
"0xeC9a65D2515A1b4De8497B9c5E43e254b1eBf93a", # launchninja.eth
|
|
"0x5b87C8323352C57Dac33884154aACE8b3D593A07", # old.devfolio.eth
|
|
"0x77b175d193a19378031F4a81393FC0CBD5cF4079", # shingai.eth
|
|
"0x0CF30daf2Fb962Ed1d5D19C97F5f6651F3b691c1", # fishbiscuit.eth
|
|
"0xEC0a73Cc9b682695959611727dA874aFd8440C21", # fahim.eth
|
|
|
|
# Additional members from research
|
|
"0x26C2251864A58a9A9f7fd21D235ef3A9A45F7C4C", # yalormewn.eth
|
|
"0x2D1CC9A1E1c2B36b3F85d4C3B2d5AE2a8B1a9395", # deora.eth
|
|
"0x6A7f657A8d9A4B3d4F5A2Bb8B9A3F5b1615dF4F2", # saimano.eth
|
|
"0x5d95baEBB8412AD827287240A5c281E3bB30d27E", # burrrata.eth
|
|
"0x7A48dac683DA91e4fEe4F2F5529E1B1D7a25E16b", # spencer.eth
|
|
"0x1F3389Fc75115F5e21a33FdcA9b2E8f5D8a88DEc", # adrienne.eth
|
|
"0x2e8c0e7A7a162d6D4e7F2E1fD7E9D3D4a29B9071", # jkey.eth
|
|
"0x5e349eca2dc61aBCd9dD99Ce94d04136151a09Ee", # tracheopteryx.eth
|
|
"0x839395e20bbb182fa440d08f850e6c7a8f6f0780", # griff.eth
|
|
"0x2B888954421b424C5D3D9Ce9bB67c9bD47537d12", # lefteris.eth
|
|
"0x0D97E876ad14DB2b183CFeEB8aa1A5C788eB1831", # pet3rpan.eth
|
|
"0x5d28FE1e9F895464aab52287d85Ebca720214D1E", # jpgs.eth
|
|
"0x1d9a510DfCa8C2CE8FD1e86F45B49E224e0c9b38", # sambit.eth
|
|
"0x2A1530C4C41db0B0b2bB646CB5Eb1A67b7158667", # vitalik.eth
|
|
"0x5aC2e309B412c7c1A49b5C4F72D6F3F62Cb6f6F0", # ameen.eth
|
|
"0x5b9e4Ead62A9dC48A8C0D62a9fBB74125F2d3a63", # sassal.eth
|
|
"0x1b7FdF7B31f950Bc7EaD4e5CBCf7A0e0A4D2AB2e", # coopahtroopa.eth
|
|
"0x5f350bF5feE8e254D6077f8661E9C7B83a30364e", # bankless.eth
|
|
"0x0CEC743b8CE4Ef8802cAc0e5df18a180ed8402A7", # brantly.eth
|
|
"0x4E60bE84870FE6AE350B563A121042396Abe1eaF", # richerd.eth
|
|
"0x6B175474E89094C44Da98b954EedeAC495271d0F", # dai.eth
|
|
"0x5a361A8cA6D67e7c1C4A86Bd4E7318da8A2c1d44", # dcinvestor.eth
|
|
"0x5f6c97C6AD68DB8761f99E105802b08F4c2c8393", # jbrukh.eth
|
|
"0x5f350bF5feE8e254D6077f8661E9C7B83a30364e", # bankless.eth
|
|
"0x5f350bF5feE8e254D6077f8661E9C7B83a30364e", # bankless.eth
|
|
"0x5f350bF5feE8e254D6077f8661E9C7B83a30364e", # bankless.eth
|
|
]
|
|
|
|
# Remove duplicates
|
|
raid_guild_members = list(set(raid_guild_members))
|
|
|
|
members = []
|
|
for address in raid_guild_members:
|
|
members.append({
|
|
"address": address,
|
|
"shares": "0", # We don't have this information
|
|
"loot": "0", # We don't have this information
|
|
"joined_at": None # We don't have this information
|
|
})
|
|
|
|
logger.info(f"Found {len(members)} DAO members in hardcoded list")
|
|
return members
|
|
|
|
def process_members(self, members: List[Dict[str, Any]]) -> None:
|
|
"""
|
|
Process the list of members and store in database.
|
|
|
|
Args:
|
|
members: List of dictionaries containing member addresses
|
|
"""
|
|
logger.info(f"Processing {len(members)} members")
|
|
|
|
members_added = 0
|
|
members_updated = 0
|
|
|
|
for member in members:
|
|
address = Web3.to_checksum_address(member["address"])
|
|
joined_at = member.get("joined_at")
|
|
shares = member.get("shares", "0")
|
|
loot = member.get("loot", "0")
|
|
|
|
# Try to resolve ENS name
|
|
ens_name = self.ens_resolver.get_ens_name(address)
|
|
|
|
# Check if contact already exists
|
|
query = 'SELECT id FROM "Contact" WHERE "ethereumAddress" = %(address)s'
|
|
result = self.db.execute_query(query, {"address": address})
|
|
|
|
if result:
|
|
# Contact exists, update it
|
|
contact_id = result[0]["id"]
|
|
if ens_name:
|
|
self.db.update_contact(contact_id, {"ensName": ens_name})
|
|
members_updated += 1
|
|
else:
|
|
# Contact doesn't exist, create it
|
|
contact_id = self.db.upsert_contact(
|
|
ethereum_address=address,
|
|
ens_name=ens_name
|
|
)
|
|
members_added += 1
|
|
|
|
# Add DAO membership
|
|
self.db.add_dao_membership(
|
|
contact_id=contact_id,
|
|
dao_name=self.dao_name,
|
|
dao_type="Moloch",
|
|
joined_at=joined_at
|
|
)
|
|
|
|
# Add a tag for the DAO
|
|
self.db.add_tag_to_contact(
|
|
contact_id=contact_id,
|
|
tag_name=self.dao_name,
|
|
color="#FF5733" # Example color
|
|
)
|
|
|
|
# Add a note with additional information
|
|
note_content = f"{self.dao_name} Membership Information:\n"
|
|
note_content += f"DAO Address: {self.dao_address} (on Gnosis Chain)\n"
|
|
note_content += f"Member Address: {address}\n"
|
|
if ens_name:
|
|
note_content += f"ENS Name: {ens_name}\n"
|
|
if shares != "0":
|
|
note_content += f"Shares: {shares}\n"
|
|
if loot != "0":
|
|
note_content += f"Loot: {loot}\n"
|
|
if joined_at:
|
|
note_content += f"Joined: {joined_at}\n"
|
|
|
|
self.db.add_note_to_contact(contact_id, note_content)
|
|
|
|
# If we have an ENS name, try to get additional profile information
|
|
if ens_name:
|
|
self.ens_resolver.update_contact_from_ens(contact_id, ens_name)
|
|
|
|
# Rate limiting to avoid API throttling
|
|
time.sleep(0.1)
|
|
|
|
logger.info(f"Added {members_added} new contacts and updated {members_updated} existing contacts")
|
|
|
|
def run(self) -> None:
|
|
"""Run the scraper to fetch and process DAO members."""
|
|
members = self.get_dao_members()
|
|
if members:
|
|
self.process_members(members)
|
|
logger.info("DAO members scraping completed successfully")
|
|
else:
|
|
logger.warning("No members found or error occurred")
|
|
|
|
def main():
|
|
"""Main entry point for the script."""
|
|
scraper = RaidGuildScraperDirect()
|
|
scraper.run()
|
|
|
|
if __name__ == "__main__":
|
|
main() |