756 lines
28 KiB
Python
Executable File
756 lines
28 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Import Public Haus Members by Querying Token Holders
|
|
|
|
This script fetches members of Public Haus DAO by querying holders of both the voting (shares)
|
|
and non-voting (loot) tokens, imports them into the database, and links them to the Public Haus DAO.
|
|
|
|
Usage:
|
|
python import_public_haus_tokens.py
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import logging
|
|
import json
|
|
import time
|
|
import requests
|
|
from typing import Dict, Any, List, Optional, Set
|
|
from web3 import Web3
|
|
from dotenv import load_dotenv
|
|
|
|
# Add parent directory to path to import utils
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from utils.db_connector import DatabaseConnector
|
|
from utils.logger import setup_logger
|
|
from utils.ens_resolver import ENSResolver
|
|
|
|
# Load environment variables
|
|
load_dotenv()
|
|
|
|
# Setup logging
|
|
logger = setup_logger("public_haus_tokens_importer")
|
|
|
|
# Constants
|
|
PUBLIC_HAUS_DAO_ID = "0xf5d6b637a9185707f52d40d452956ca49018247a" # Public Haus DAO ID on Optimism
|
|
SHARES_TOKEN_ADDRESS = "0x4950c436F69c8b4F80f688edc814C5bA84Aa70f5" # Voting token (shares)
|
|
LOOT_TOKEN_ADDRESS = "0xab6033E3EC2144FB279fe68dA92B7aC6a42Da6d8" # Non-voting token (loot)
|
|
|
|
# Optimism Etherscan API
|
|
OPTIMISM_ETHERSCAN_API_URL = "https://api-optimistic.etherscan.io/api"
|
|
|
|
# ERC20 ABI (minimal for balance checking)
|
|
ERC20_ABI = [
|
|
{
|
|
"constant": True,
|
|
"inputs": [{"name": "_owner", "type": "address"}],
|
|
"name": "balanceOf",
|
|
"outputs": [{"name": "balance", "type": "uint256"}],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"constant": True,
|
|
"inputs": [],
|
|
"name": "totalSupply",
|
|
"outputs": [{"name": "", "type": "uint256"}],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"constant": True,
|
|
"inputs": [],
|
|
"name": "name",
|
|
"outputs": [{"name": "", "type": "string"}],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"constant": True,
|
|
"inputs": [],
|
|
"name": "symbol",
|
|
"outputs": [{"name": "", "type": "string"}],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"constant": True,
|
|
"inputs": [],
|
|
"name": "decimals",
|
|
"outputs": [{"name": "", "type": "uint8"}],
|
|
"type": "function"
|
|
}
|
|
]
|
|
|
|
class PublicHausTokensImporter:
|
|
"""Importer for Public Haus DAO members based on token holdings"""
|
|
|
|
def __init__(self):
|
|
"""Initialize the importer"""
|
|
# Initialize database
|
|
self.db = DatabaseConnector()
|
|
|
|
# Initialize Web3
|
|
optimism_rpc_url = os.getenv("OPTIMISM_RPC_URL")
|
|
if not optimism_rpc_url:
|
|
raise ValueError("OPTIMISM_RPC_URL environment variable not set")
|
|
|
|
self.web3 = Web3(Web3.HTTPProvider(optimism_rpc_url))
|
|
if not self.web3.is_connected():
|
|
raise ValueError("Failed to connect to Optimism RPC")
|
|
|
|
logger.info(f"Connected to Optimism: {self.web3.is_connected()}")
|
|
|
|
# Initialize token contracts
|
|
self.shares_token = self.web3.eth.contract(
|
|
address=self.web3.to_checksum_address(SHARES_TOKEN_ADDRESS),
|
|
abi=ERC20_ABI
|
|
)
|
|
|
|
self.loot_token = self.web3.eth.contract(
|
|
address=self.web3.to_checksum_address(LOOT_TOKEN_ADDRESS),
|
|
abi=ERC20_ABI
|
|
)
|
|
|
|
# Get Etherscan API key
|
|
self.etherscan_api_key = os.getenv("OPTIMISM_ETHERSCAN_API_KEY")
|
|
if not self.etherscan_api_key:
|
|
logger.warning("OPTIMISM_ETHERSCAN_API_KEY not set, using API without key (rate limited)")
|
|
self.etherscan_api_key = ""
|
|
else:
|
|
logger.info("Using Optimism Etherscan API key")
|
|
|
|
# Initialize ENS resolver for Ethereum mainnet
|
|
ethereum_rpc_url = os.getenv("ETHEREUM_RPC_URL", "https://eth-mainnet.g.alchemy.com/v2/1fkIfqUX_MoHhd4Qqnu9UItM8Fc4Ls2q")
|
|
self.eth_web3 = Web3(Web3.HTTPProvider(ethereum_rpc_url))
|
|
if not self.eth_web3.is_connected():
|
|
logger.warning("Failed to connect to Ethereum RPC for ENS resolution")
|
|
self.ens_resolver = None
|
|
else:
|
|
logger.info(f"Connected to Ethereum for ENS resolution: {self.eth_web3.is_connected()}")
|
|
self.ens_resolver = ENSResolver(self.eth_web3)
|
|
|
|
# Register data source
|
|
self.data_source_id = self.register_data_source()
|
|
|
|
# Initialize scraping job
|
|
self.job_id = self.db.create_scraping_job(
|
|
source_name="Public Haus DAO Tokens",
|
|
status="running"
|
|
)
|
|
logger.info(f"Created scraping job with ID: {self.job_id}")
|
|
|
|
def register_data_source(self) -> str:
|
|
"""Register the Public Haus data source in the database"""
|
|
return self.db.upsert_data_source(
|
|
name="Public Haus DAO Tokens",
|
|
source_type="blockchain",
|
|
description="Public Haus DAO members identified by token holdings"
|
|
)
|
|
|
|
def get_token_info(self, token_contract, token_name) -> Dict[str, Any]:
|
|
"""
|
|
Get information about a token
|
|
|
|
Args:
|
|
token_contract: Web3 contract instance
|
|
token_name: Name of the token for logging
|
|
|
|
Returns:
|
|
Token information
|
|
"""
|
|
try:
|
|
name = token_contract.functions.name().call()
|
|
symbol = token_contract.functions.symbol().call()
|
|
decimals = token_contract.functions.decimals().call()
|
|
total_supply = token_contract.functions.totalSupply().call()
|
|
|
|
token_info = {
|
|
"name": name,
|
|
"symbol": symbol,
|
|
"decimals": decimals,
|
|
"totalSupply": total_supply
|
|
}
|
|
|
|
logger.info(f"{token_name} info: {name} ({symbol})")
|
|
logger.info(f"{token_name} total supply: {total_supply / (10 ** decimals):.2f} {symbol}")
|
|
|
|
return token_info
|
|
except Exception as e:
|
|
logger.error(f"Error getting {token_name} info via Web3: {e}")
|
|
|
|
# Try Etherscan API as fallback
|
|
try:
|
|
token_address = token_contract.address
|
|
params = {
|
|
"module": "token",
|
|
"action": "tokeninfo",
|
|
"contractaddress": token_address,
|
|
"apikey": self.etherscan_api_key
|
|
}
|
|
|
|
response = requests.get(OPTIMISM_ETHERSCAN_API_URL, params=params)
|
|
data = response.json()
|
|
|
|
if data["status"] == "1":
|
|
token_info = data["result"][0]
|
|
logger.info(f"{token_name} info from Etherscan: {token_info.get('name')} ({token_info.get('symbol')})")
|
|
return {
|
|
"name": token_info.get("name", f"Public Haus {token_name}"),
|
|
"symbol": token_info.get("symbol", token_name.upper()),
|
|
"decimals": int(token_info.get("decimals", 18)),
|
|
"totalSupply": int(token_info.get("totalSupply", "0"))
|
|
}
|
|
except Exception as etherscan_error:
|
|
logger.error(f"Error getting {token_name} info via Etherscan: {etherscan_error}")
|
|
|
|
# Return default values if both methods fail
|
|
return {
|
|
"name": f"Public Haus {token_name}",
|
|
"symbol": token_name.upper(),
|
|
"decimals": 18,
|
|
"totalSupply": 0
|
|
}
|
|
|
|
def fetch_token_holders_via_etherscan(self, token_address, token_name, decimals) -> List[Dict[str, Any]]:
|
|
"""
|
|
Fetch holders of a token using Etherscan API
|
|
|
|
Args:
|
|
token_address: Address of the token
|
|
token_name: Name of the token for logging
|
|
decimals: Token decimals
|
|
|
|
Returns:
|
|
List of token holders with their balances
|
|
"""
|
|
try:
|
|
# Get token holders from Etherscan
|
|
params = {
|
|
"module": "token",
|
|
"action": "tokenholderlist",
|
|
"contractaddress": token_address,
|
|
"page": 1,
|
|
"offset": 100, # Get up to 100 holders
|
|
"apikey": self.etherscan_api_key
|
|
}
|
|
|
|
response = requests.get(OPTIMISM_ETHERSCAN_API_URL, params=params)
|
|
data = response.json()
|
|
|
|
holders = []
|
|
|
|
if data["status"] == "1":
|
|
for holder in data["result"]:
|
|
address = holder["address"]
|
|
balance = int(holder["TokenHolderQuantity"])
|
|
|
|
# Skip zero balances
|
|
if balance > 0:
|
|
holders.append({
|
|
"address": address,
|
|
"balance": balance,
|
|
"balanceFormatted": balance / (10 ** decimals),
|
|
"tokenType": token_name
|
|
})
|
|
|
|
logger.info(f"Found {len(holders)} {token_name} holders with non-zero balance via Etherscan")
|
|
return holders
|
|
else:
|
|
logger.warning(f"Error getting {token_name} holders from Etherscan: {data.get('message')}")
|
|
return []
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching {token_name} holders via Etherscan: {e}")
|
|
return []
|
|
|
|
def fetch_token_transfers_via_etherscan(self, token_address, token_name, decimals) -> List[Dict[str, Any]]:
|
|
"""
|
|
Fetch token transfers using Etherscan API and extract unique addresses
|
|
|
|
Args:
|
|
token_address: Address of the token
|
|
token_name: Name of the token for logging
|
|
decimals: Token decimals
|
|
|
|
Returns:
|
|
List of token holders with their balances
|
|
"""
|
|
try:
|
|
# Get token transfers from Etherscan
|
|
params = {
|
|
"module": "account",
|
|
"action": "tokentx",
|
|
"contractaddress": token_address,
|
|
"page": 1,
|
|
"offset": 1000, # Get up to 1000 transfers
|
|
"sort": "desc",
|
|
"apikey": self.etherscan_api_key
|
|
}
|
|
|
|
response = requests.get(OPTIMISM_ETHERSCAN_API_URL, params=params)
|
|
data = response.json()
|
|
|
|
addresses = set()
|
|
|
|
if data["status"] == "1":
|
|
for tx in data["result"]:
|
|
addresses.add(tx["to"])
|
|
addresses.add(tx["from"])
|
|
|
|
# Remove zero address
|
|
if "0x0000000000000000000000000000000000000000" in addresses:
|
|
addresses.remove("0x0000000000000000000000000000000000000000")
|
|
|
|
# Create holder objects
|
|
holders = []
|
|
for address in addresses:
|
|
holders.append({
|
|
"address": address,
|
|
"balance": 1, # We don't know the actual balance
|
|
"balanceFormatted": 1,
|
|
"tokenType": token_name
|
|
})
|
|
|
|
logger.info(f"Found {len(holders)} unique addresses from {token_name} transfers via Etherscan")
|
|
return holders
|
|
else:
|
|
logger.warning(f"Error getting {token_name} transfers from Etherscan: {data.get('message')}")
|
|
return []
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching {token_name} transfers via Etherscan: {e}")
|
|
return []
|
|
|
|
def fetch_token_holders_via_web3(self, token_contract, token_name, decimals) -> List[Dict[str, Any]]:
|
|
"""
|
|
Fetch holders of a token by checking balances of known addresses
|
|
|
|
Args:
|
|
token_contract: Web3 contract instance
|
|
token_name: Name of the token for logging
|
|
decimals: Token decimals
|
|
|
|
Returns:
|
|
List of token holders with their balances
|
|
"""
|
|
try:
|
|
# Get the latest block number
|
|
latest_block = self.web3.eth.block_number
|
|
|
|
# Try to get some known addresses from recent transactions to the token contract
|
|
known_addresses = set()
|
|
|
|
# Check the last 100 blocks for transactions to the token contract
|
|
for block_num in range(max(0, latest_block - 100), latest_block + 1):
|
|
try:
|
|
block = self.web3.eth.get_block(block_num, full_transactions=True)
|
|
for tx in block.transactions:
|
|
if tx.to and tx.to.lower() == token_contract.address.lower():
|
|
known_addresses.add(tx['from'])
|
|
except Exception as e:
|
|
logger.warning(f"Error getting block {block_num}: {e}")
|
|
continue
|
|
|
|
# Add the DAO address as a known address
|
|
known_addresses.add(self.web3.to_checksum_address(PUBLIC_HAUS_DAO_ID))
|
|
|
|
# Check balances for known addresses
|
|
holders = []
|
|
for address in known_addresses:
|
|
try:
|
|
balance = token_contract.functions.balanceOf(address).call()
|
|
|
|
# Only include addresses with non-zero balance
|
|
if balance > 0:
|
|
holders.append({
|
|
"address": address,
|
|
"balance": balance,
|
|
"balanceFormatted": balance / (10 ** decimals),
|
|
"tokenType": token_name
|
|
})
|
|
except Exception as e:
|
|
logger.error(f"Error checking {token_name} balance for {address}: {e}")
|
|
|
|
logger.info(f"Found {len(holders)} {token_name} holders with non-zero balance via Web3")
|
|
return holders
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching {token_name} holders via Web3: {e}")
|
|
return []
|
|
|
|
def fetch_all_token_holders(self) -> List[Dict[str, Any]]:
|
|
"""
|
|
Fetch holders of both shares and loot tokens
|
|
|
|
Returns:
|
|
List of token holders with their balances
|
|
"""
|
|
all_holders = []
|
|
|
|
# Get token info
|
|
shares_info = self.get_token_info(self.shares_token, "Shares")
|
|
loot_info = self.get_token_info(self.loot_token, "Loot")
|
|
|
|
shares_decimals = shares_info["decimals"]
|
|
loot_decimals = loot_info["decimals"]
|
|
|
|
# Try different methods to get token holders
|
|
|
|
# 1. Try Etherscan tokenholderlist endpoint
|
|
shares_holders = self.fetch_token_holders_via_etherscan(
|
|
SHARES_TOKEN_ADDRESS, "Shares", shares_decimals
|
|
)
|
|
loot_holders = self.fetch_token_holders_via_etherscan(
|
|
LOOT_TOKEN_ADDRESS, "Loot", loot_decimals
|
|
)
|
|
|
|
# 2. If that fails, try getting transfers
|
|
if not shares_holders:
|
|
shares_holders = self.fetch_token_transfers_via_etherscan(
|
|
SHARES_TOKEN_ADDRESS, "Shares", shares_decimals
|
|
)
|
|
|
|
if not loot_holders:
|
|
loot_holders = self.fetch_token_transfers_via_etherscan(
|
|
LOOT_TOKEN_ADDRESS, "Loot", loot_decimals
|
|
)
|
|
|
|
# 3. If that fails, try Web3
|
|
if not shares_holders:
|
|
shares_holders = self.fetch_token_holders_via_web3(
|
|
self.shares_token, "Shares", shares_decimals
|
|
)
|
|
|
|
if not loot_holders:
|
|
loot_holders = self.fetch_token_holders_via_web3(
|
|
self.loot_token, "Loot", loot_decimals
|
|
)
|
|
|
|
# Combine holders
|
|
all_holders.extend(shares_holders)
|
|
all_holders.extend(loot_holders)
|
|
|
|
# If we still don't have any holders, use the DAO address itself
|
|
if not all_holders:
|
|
logger.warning("No token holders found, using DAO address as fallback")
|
|
all_holders.append({
|
|
"address": PUBLIC_HAUS_DAO_ID,
|
|
"balance": 1,
|
|
"balanceFormatted": 1,
|
|
"tokenType": "Fallback"
|
|
})
|
|
|
|
return all_holders
|
|
|
|
def merge_holders(self, holders: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
"""
|
|
Merge holders by address, combining shares and loot
|
|
|
|
Args:
|
|
holders: List of token holders
|
|
|
|
Returns:
|
|
List of merged holders
|
|
"""
|
|
merged = {}
|
|
|
|
for holder in holders:
|
|
address = holder["address"]
|
|
balance = holder["balance"]
|
|
token_type = holder["tokenType"]
|
|
|
|
if address not in merged:
|
|
merged[address] = {
|
|
"address": address,
|
|
"shares": 0,
|
|
"sharesFormatted": 0,
|
|
"loot": 0,
|
|
"lootFormatted": 0,
|
|
"dao": "Public Haus"
|
|
}
|
|
|
|
if token_type == "Shares":
|
|
merged[address]["shares"] = balance
|
|
merged[address]["sharesFormatted"] = holder["balanceFormatted"]
|
|
elif token_type == "Loot":
|
|
merged[address]["loot"] = balance
|
|
merged[address]["lootFormatted"] = holder["balanceFormatted"]
|
|
else:
|
|
# Fallback
|
|
merged[address]["shares"] = balance
|
|
merged[address]["sharesFormatted"] = holder["balanceFormatted"]
|
|
|
|
return list(merged.values())
|
|
|
|
def resolve_ens_name(self, address: str) -> Optional[str]:
|
|
"""
|
|
Resolve ENS name for an Ethereum address
|
|
|
|
Args:
|
|
address: Ethereum address to resolve
|
|
|
|
Returns:
|
|
ENS name if found, None otherwise
|
|
"""
|
|
if not self.ens_resolver:
|
|
return None
|
|
|
|
try:
|
|
ens_name = self.ens_resolver.get_ens_name(address)
|
|
if ens_name:
|
|
logger.info(f"Resolved ENS name for {address}: {ens_name}")
|
|
return ens_name
|
|
except Exception as e:
|
|
logger.error(f"Error resolving ENS name for {address}: {e}")
|
|
return None
|
|
|
|
def process_holder(self, holder: Dict[str, Any]) -> Optional[str]:
|
|
"""
|
|
Process a token holder and import into the database
|
|
|
|
Args:
|
|
holder: Token holder information
|
|
|
|
Returns:
|
|
Contact ID if successful, None otherwise
|
|
"""
|
|
try:
|
|
# Extract holder information
|
|
address = holder["address"]
|
|
shares = holder["shares"]
|
|
shares_formatted = holder["sharesFormatted"]
|
|
loot = holder["loot"]
|
|
loot_formatted = holder["lootFormatted"]
|
|
dao_name = holder["dao"]
|
|
|
|
# Check if contact exists
|
|
query = 'SELECT id, name, "ensName" FROM "Contact" WHERE "ethereumAddress" ILIKE %(address)s'
|
|
existing_contacts = self.db.execute_query(query, {"address": address})
|
|
|
|
contact_id = None
|
|
|
|
if existing_contacts:
|
|
# Use existing contact
|
|
contact_id = existing_contacts[0]["id"]
|
|
logger.info(f"Found existing contact {contact_id} for address {address}")
|
|
else:
|
|
# Create new contact
|
|
contact_id = self.db.upsert_contact(
|
|
ethereum_address=address,
|
|
ens_name=None
|
|
)
|
|
logger.info(f"Created new contact {contact_id} for address {address}")
|
|
|
|
# Add DAO membership
|
|
self.db.execute_update(
|
|
"""
|
|
INSERT INTO "DaoMembership" (id, "contactId", "daoName", "daoType", "createdAt", "updatedAt")
|
|
VALUES (gen_random_uuid(), %(contact_id)s, %(dao_name)s, %(dao_type)s, NOW(), NOW())
|
|
ON CONFLICT ("contactId", "daoName")
|
|
DO UPDATE SET
|
|
"updatedAt" = NOW()
|
|
""",
|
|
{
|
|
"contact_id": contact_id,
|
|
"dao_name": dao_name,
|
|
"dao_type": "Moloch V3"
|
|
}
|
|
)
|
|
|
|
# Add note about membership with token holdings
|
|
note_content = f"Public Haus DAO Member\nShares: {shares_formatted}\nLoot: {loot_formatted}"
|
|
|
|
self.db.add_note_to_contact(
|
|
contact_id=contact_id,
|
|
content=note_content
|
|
)
|
|
|
|
# Add tags for the DAO and token holdings
|
|
self.db.add_tag_to_contact(
|
|
contact_id=contact_id,
|
|
tag_name=dao_name
|
|
)
|
|
|
|
if shares > 0:
|
|
self.db.add_tag_to_contact(
|
|
contact_id=contact_id,
|
|
tag_name=f"{dao_name} Voting Member"
|
|
)
|
|
|
|
if loot > 0:
|
|
self.db.add_tag_to_contact(
|
|
contact_id=contact_id,
|
|
tag_name=f"{dao_name} Non-Voting Member"
|
|
)
|
|
|
|
# Link to data source
|
|
self.db.link_contact_to_data_source(contact_id, self.data_source_id)
|
|
|
|
return contact_id
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing holder {holder.get('address')}: {e}")
|
|
return None
|
|
|
|
def process_address(self, address: str, shares_balance: float = 0, loot_balance: float = 0) -> Optional[str]:
|
|
"""
|
|
Process a single address, creating or updating a contact and linking to the DAO
|
|
|
|
Args:
|
|
address: Ethereum address
|
|
shares_balance: Balance of shares token
|
|
loot_balance: Balance of loot token
|
|
|
|
Returns:
|
|
Contact ID if successful, None otherwise
|
|
"""
|
|
try:
|
|
# Check if contact already exists
|
|
query = 'SELECT id, name, "ensName" FROM "Contact" WHERE "ethereumAddress" ILIKE %(address)s'
|
|
result = self.db.execute_query(query, {"address": address})
|
|
|
|
contact_id = None
|
|
is_new = False
|
|
|
|
# Resolve ENS name if needed
|
|
ens_name = None
|
|
if not result or not result[0].get("ensName"):
|
|
ens_name = self.resolve_ens_name(address)
|
|
|
|
if result:
|
|
# Contact exists, get ID
|
|
contact_id = result[0]["id"]
|
|
logger.info(f"Found existing contact for {address}: {contact_id}")
|
|
|
|
# Update ENS name if we found one and it's not already set
|
|
if ens_name and not result[0].get("ensName"):
|
|
self.db.update_contact(contact_id, {"ensName": ens_name})
|
|
logger.info(f"Updated ENS name for contact {contact_id}: {ens_name}")
|
|
else:
|
|
# Create new contact
|
|
# Use ENS name as contact name if available, otherwise use address
|
|
contact_name = ens_name.split('.')[0] if ens_name else f"ETH_{address[:8]}"
|
|
|
|
contact_id = self.db.upsert_contact(
|
|
ethereum_address=address,
|
|
ens_name=ens_name
|
|
)
|
|
logger.info(f"Created new contact for {address}: {contact_id}")
|
|
is_new = True
|
|
|
|
# Add DAO membership
|
|
self.db.add_dao_membership(
|
|
contact_id=contact_id,
|
|
dao_name="Public Haus",
|
|
dao_type="Moloch V3"
|
|
)
|
|
|
|
# Add note about membership with token holdings
|
|
note_content = f"Public Haus DAO Member\nShares: {shares_balance}\nLoot: {loot_balance}"
|
|
|
|
self.db.add_note_to_contact(
|
|
contact_id=contact_id,
|
|
content=note_content
|
|
)
|
|
|
|
# Add tags for the DAO and token holdings
|
|
self.db.add_tag_to_contact(
|
|
contact_id=contact_id,
|
|
tag_name="Public Haus"
|
|
)
|
|
|
|
if shares_balance > 0:
|
|
self.db.add_tag_to_contact(
|
|
contact_id=contact_id,
|
|
tag_name="Public Haus Voting Member"
|
|
)
|
|
|
|
if loot_balance > 0:
|
|
self.db.add_tag_to_contact(
|
|
contact_id=contact_id,
|
|
tag_name="Public Haus Non-Voting Member"
|
|
)
|
|
|
|
# Link to data source
|
|
self.db.link_contact_to_data_source(contact_id, self.data_source_id)
|
|
|
|
return contact_id
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing address {address}: {e}")
|
|
return None
|
|
|
|
def run(self) -> int:
|
|
"""
|
|
Run the importer
|
|
|
|
Returns:
|
|
Number of holders imported
|
|
"""
|
|
try:
|
|
# Fetch token holders
|
|
all_holders = self.fetch_all_token_holders()
|
|
|
|
# Merge holders by address
|
|
merged_holders = self.merge_holders(all_holders)
|
|
|
|
if not merged_holders:
|
|
logger.info("No token holders found")
|
|
self.db.update_scraping_job(self.job_id, "completed")
|
|
return 0
|
|
|
|
# Process holders
|
|
imported_count = 0
|
|
existing_count = 0
|
|
|
|
for holder in merged_holders:
|
|
try:
|
|
contact_id = self.process_address(
|
|
address=holder["address"],
|
|
shares_balance=holder["shares"],
|
|
loot_balance=holder["loot"]
|
|
)
|
|
if contact_id:
|
|
imported_count += 1
|
|
except Exception as e:
|
|
logger.exception(f"Error processing holder {holder.get('address')}: {e}")
|
|
|
|
# Add a small delay to avoid overwhelming the database
|
|
time.sleep(0.1)
|
|
|
|
# Complete the scraping job
|
|
self.db.update_scraping_job(
|
|
self.job_id,
|
|
"completed",
|
|
records_processed=len(merged_holders),
|
|
records_added=imported_count,
|
|
records_updated=existing_count
|
|
)
|
|
|
|
logger.info(f"Imported {imported_count} holders out of {len(merged_holders)} processed")
|
|
|
|
# Run ENS resolution for any contacts that don't have ENS names
|
|
if self.ens_resolver:
|
|
logger.info("Running ENS resolution for contacts without ENS names...")
|
|
from utils.resolve_ens_names import ENSResolver as BatchENSResolver
|
|
batch_resolver = BatchENSResolver()
|
|
batch_resolver.run(batch_size=50, delay_seconds=0.5)
|
|
|
|
return imported_count
|
|
|
|
except Exception as e:
|
|
# Update the scraping job with error
|
|
self.db.update_scraping_job(self.job_id, "failed", error_message=str(e))
|
|
logger.exception(f"Error importing holders: {e}")
|
|
raise
|
|
|
|
def main():
|
|
"""Main function"""
|
|
try:
|
|
importer = PublicHausTokensImporter()
|
|
imported_count = importer.run()
|
|
logger.info(f"Import completed successfully. Imported {imported_count} token holders.")
|
|
return 0
|
|
except Exception as e:
|
|
logger.exception(f"Error importing token holders: {e}")
|
|
return 1
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main()) |