333 lines
11 KiB
Python
333 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Import Public Haus Members from Optimism Blockchain
|
|
|
|
This script fetches members of Public Haus DAO by directly querying the Optimism blockchain,
|
|
imports them into the database, and links them to the Public Haus DAO.
|
|
|
|
Usage:
|
|
python import_public_haus_members_web3.py
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import logging
|
|
import json
|
|
import time
|
|
from typing import Dict, Any, List, Optional
|
|
from web3 import Web3
|
|
from dotenv import load_dotenv
|
|
|
|
# Add parent directory to path to import utils
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from utils.db_connector import DatabaseConnector
|
|
from utils.logger import setup_logger
|
|
|
|
# Load environment variables
|
|
load_dotenv()
|
|
|
|
# Setup logging
|
|
logger = setup_logger("public_haus_importer")
|
|
|
|
# Constants
|
|
PUBLIC_HAUS_DAO_ID = "0xf5d6b637a9185707f52d40d452956ca49018247a" # Public Haus DAO ID on Optimism
|
|
|
|
# Moloch DAO V3 ABI (partial, only what we need)
|
|
MOLOCH_V3_ABI = [
|
|
{
|
|
"inputs": [{"internalType": "address", "name": "memberAddress", "type": "address"}],
|
|
"name": "members",
|
|
"outputs": [
|
|
{"internalType": "address", "name": "delegateKey", "type": "address"},
|
|
{"internalType": "uint256", "name": "shares", "type": "uint256"},
|
|
{"internalType": "uint256", "name": "loot", "type": "uint256"},
|
|
{"internalType": "bool", "name": "exists", "type": "bool"},
|
|
{"internalType": "uint256", "name": "highestIndexYesVote", "type": "uint256"},
|
|
{"internalType": "uint256", "name": "jailed", "type": "uint256"}
|
|
],
|
|
"stateMutability": "view",
|
|
"type": "function"
|
|
},
|
|
{
|
|
"inputs": [],
|
|
"name": "getMemberAddresses",
|
|
"outputs": [{"internalType": "address[]", "name": "", "type": "address[]"}],
|
|
"stateMutability": "view",
|
|
"type": "function"
|
|
},
|
|
{
|
|
"inputs": [],
|
|
"name": "totalShares",
|
|
"outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}],
|
|
"stateMutability": "view",
|
|
"type": "function"
|
|
},
|
|
{
|
|
"inputs": [],
|
|
"name": "totalLoot",
|
|
"outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}],
|
|
"stateMutability": "view",
|
|
"type": "function"
|
|
}
|
|
]
|
|
|
|
class PublicHausImporter:
|
|
"""Importer for Public Haus members from Optimism blockchain"""
|
|
|
|
def __init__(self):
|
|
"""Initialize the importer"""
|
|
# Initialize database
|
|
self.db = DatabaseConnector()
|
|
|
|
# Initialize Web3
|
|
optimism_rpc_url = os.getenv("OPTIMISM_RPC_URL")
|
|
if not optimism_rpc_url:
|
|
raise ValueError("OPTIMISM_RPC_URL environment variable not set")
|
|
|
|
self.web3 = Web3(Web3.HTTPProvider(optimism_rpc_url))
|
|
if not self.web3.is_connected():
|
|
raise ValueError("Failed to connect to Optimism RPC")
|
|
|
|
logger.info(f"Connected to Optimism: {self.web3.is_connected()}")
|
|
|
|
# Initialize contract
|
|
self.contract = self.web3.eth.contract(
|
|
address=self.web3.to_checksum_address(PUBLIC_HAUS_DAO_ID),
|
|
abi=MOLOCH_V3_ABI
|
|
)
|
|
|
|
# Register data source
|
|
self.data_source_id = self.register_data_source()
|
|
|
|
def register_data_source(self) -> str:
|
|
"""Register the Public Haus data source in the database"""
|
|
return self.db.upsert_data_source(
|
|
name="Public Haus DAO Blockchain",
|
|
source_type="blockchain",
|
|
description="Public Haus DAO members from Optimism blockchain"
|
|
)
|
|
|
|
def fetch_dao_info(self) -> Dict[str, Any]:
|
|
"""
|
|
Fetch Public Haus DAO information from the blockchain
|
|
|
|
Returns:
|
|
DAO information
|
|
"""
|
|
try:
|
|
# Get total shares and loot
|
|
total_shares = self.contract.functions.totalShares().call()
|
|
total_loot = self.contract.functions.totalLoot().call()
|
|
|
|
dao_info = {
|
|
"id": PUBLIC_HAUS_DAO_ID,
|
|
"name": "Public Haus",
|
|
"totalShares": total_shares,
|
|
"totalLoot": total_loot
|
|
}
|
|
|
|
logger.info(f"Fetched DAO info: Public Haus")
|
|
logger.info(f"Total Shares: {total_shares}")
|
|
logger.info(f"Total Loot: {total_loot}")
|
|
|
|
return dao_info
|
|
except Exception as e:
|
|
logger.error(f"Error fetching DAO info: {e}")
|
|
raise
|
|
|
|
def fetch_members(self) -> List[Dict[str, Any]]:
|
|
"""
|
|
Fetch Public Haus members from the blockchain
|
|
|
|
Returns:
|
|
List of member data
|
|
"""
|
|
try:
|
|
# Get member addresses
|
|
member_addresses = self.contract.functions.getMemberAddresses().call()
|
|
logger.info(f"Fetched {len(member_addresses)} member addresses")
|
|
|
|
members = []
|
|
|
|
# Get member details
|
|
for address in member_addresses:
|
|
try:
|
|
member = self.contract.functions.members(address).call()
|
|
|
|
# Extract member data
|
|
delegate_key, shares, loot, exists, highest_index_yes_vote, jailed = member
|
|
|
|
# Skip if not a member
|
|
if not exists:
|
|
continue
|
|
|
|
# Create member object
|
|
member_data = {
|
|
"memberAddress": address,
|
|
"delegateKey": delegate_key,
|
|
"shares": shares,
|
|
"loot": loot,
|
|
"jailed": jailed > 0
|
|
}
|
|
|
|
members.append(member_data)
|
|
except Exception as e:
|
|
logger.error(f"Error fetching member {address}: {e}")
|
|
|
|
logger.info(f"Fetched {len(members)} members with details")
|
|
|
|
return members
|
|
except Exception as e:
|
|
logger.error(f"Error fetching members: {e}")
|
|
raise
|
|
|
|
def process_member(self, member: Dict[str, Any]) -> Optional[str]:
|
|
"""
|
|
Process a single member and import into database
|
|
|
|
Args:
|
|
member: Member data from the blockchain
|
|
|
|
Returns:
|
|
Contact ID if successful, None otherwise
|
|
"""
|
|
# Extract member data
|
|
address = member["memberAddress"]
|
|
shares = int(member["shares"])
|
|
loot = int(member["loot"])
|
|
delegate_key = member["delegateKey"]
|
|
jailed = member["jailed"]
|
|
|
|
# Skip if no address
|
|
if not address:
|
|
logger.warning(f"Member has no address: {member}")
|
|
return None
|
|
|
|
# Check if contact already exists
|
|
query = 'SELECT id, name, "ensName" FROM "Contact" WHERE "ethereumAddress" ILIKE %(address)s'
|
|
existing_contacts = self.db.execute_query(query, {"address": address})
|
|
|
|
contact_id = None
|
|
|
|
if existing_contacts:
|
|
# Use existing contact
|
|
contact_id = existing_contacts[0]["id"]
|
|
logger.info(f"Found existing contact {contact_id} for address {address}")
|
|
else:
|
|
# Create new contact
|
|
contact_data = {
|
|
"ethereumAddress": address,
|
|
"name": f"Public Haus Member {address[:8]}", # Default name
|
|
}
|
|
|
|
contact_id = self.db.upsert_contact(contact_data)
|
|
logger.info(f"Created new contact {contact_id} for address {address}")
|
|
|
|
# Add DAO membership
|
|
self.db.execute_update(
|
|
"""
|
|
INSERT INTO "DaoMembership" ("contactId", "daoName", "shares", "loot", "delegatingTo")
|
|
VALUES (%(contact_id)s, %(dao_name)s, %(shares)s, %(loot)s, %(delegating_to)s)
|
|
ON CONFLICT ("contactId", "daoName")
|
|
DO UPDATE SET
|
|
"shares" = %(shares)s,
|
|
"loot" = %(loot)s,
|
|
"delegatingTo" = %(delegating_to)s,
|
|
"updatedAt" = NOW()
|
|
""",
|
|
{
|
|
"contact_id": contact_id,
|
|
"dao_name": "Public Haus",
|
|
"shares": shares,
|
|
"loot": loot,
|
|
"delegating_to": delegate_key if delegate_key != address else None
|
|
}
|
|
)
|
|
|
|
# Add note about membership
|
|
note_content = f"Public Haus DAO Member\nShares: {shares}\nLoot: {loot}"
|
|
if delegate_key != address:
|
|
note_content += f"\nDelegating to: {delegate_key}"
|
|
if jailed:
|
|
note_content += "\nJailed: Yes"
|
|
|
|
self.db.add_note_to_contact(
|
|
contact_id=contact_id,
|
|
content=note_content,
|
|
source="Public Haus DAO Blockchain"
|
|
)
|
|
|
|
# Link to data source
|
|
self.db.link_contact_to_data_source(contact_id, self.data_source_id)
|
|
|
|
return contact_id
|
|
|
|
def run(self) -> int:
|
|
"""
|
|
Run the importer
|
|
|
|
Returns:
|
|
Number of members imported
|
|
"""
|
|
# Create a scraping job
|
|
job_id = self.db.create_scraping_job("Public Haus DAO Importer", "running")
|
|
logger.info(f"Created scraping job with ID: {job_id}")
|
|
|
|
try:
|
|
# Fetch DAO info
|
|
dao_info = self.fetch_dao_info()
|
|
|
|
# Fetch members
|
|
members = self.fetch_members()
|
|
|
|
if not members:
|
|
logger.info("No members found")
|
|
self.db.update_scraping_job(job_id, "completed")
|
|
return 0
|
|
|
|
# Process members
|
|
imported_count = 0
|
|
existing_count = 0
|
|
|
|
for member in members:
|
|
try:
|
|
contact_id = self.process_member(member)
|
|
if contact_id:
|
|
imported_count += 1
|
|
except Exception as e:
|
|
logger.exception(f"Error processing member {member.get('memberAddress')}: {e}")
|
|
|
|
# Add a small delay to avoid overwhelming the database
|
|
time.sleep(0.1)
|
|
|
|
# Complete the scraping job
|
|
self.db.update_scraping_job(
|
|
job_id,
|
|
"completed",
|
|
records_processed=len(members),
|
|
records_added=imported_count,
|
|
records_updated=existing_count
|
|
)
|
|
|
|
logger.info(f"Imported {imported_count} members out of {len(members)} processed")
|
|
return imported_count
|
|
|
|
except Exception as e:
|
|
# Update the scraping job with error
|
|
self.db.update_scraping_job(job_id, "failed", error_message=str(e))
|
|
logger.exception(f"Error importing members: {e}")
|
|
raise
|
|
|
|
def main():
|
|
"""Main function"""
|
|
try:
|
|
importer = PublicHausImporter()
|
|
imported_count = importer.run()
|
|
logger.info(f"Import completed successfully. Imported {imported_count} members.")
|
|
return 0
|
|
except Exception as e:
|
|
logger.exception(f"Error importing members: {e}")
|
|
return 1
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main()) |