Find Minting Data (OpenSea)

"""
Update Parameters Here
"""
COLLECTION_NAME = "Quaks"
CONTRACT = "0x07bbdaf30e89ea3ecf6cadc80d6e7c4b0843c729"
BEFORE_TIME = "2021-09-02T00:00:00"  # One day after the last mint (e.g. https://etherscan.io/tx/0x206c846d0d1739faa9835e16ff419d15708a558357a9413619e65dacf095ac7a)

# these should usually stay the same
METHOD = "raritytools"

"""
Created on Tue Sep 14 20:17:07 2021
mint data. Doesn't work when Opensea's API is being shitty
@author: nbax1, slight modifications by mdigi14
"""

import pandas as pd

from honestnft_utils import config
from honestnft_utils import constants
from honestnft_utils import opensea


"""
Helper Functions
"""


def get_mint_events(
    contract: str, before_time: str, rarity_db: pd.DataFrame
) -> pd.DataFrame:
    data = opensea.get_opensea_events(
        contract_address=contract,
        account_address=constants.MINT_ADDRESS,
        event_type="transfer",
        occurred_before=before_time,
    )

    df = pd.json_normalize(data)

    df = df.loc[df["from_account.address"] == constants.MINT_ADDRESS]
    df_rar = pd.DataFrame(rarity_db)

    os_tokens = df["asset.token_id"].astype(int).tolist()
    rar_tokens = df_rar["TOKEN_ID"].astype(int).tolist()

    set1 = set(rar_tokens)
    set2 = set(os_tokens)

    missing_tokens = list(sorted(set1 - set2))
    if missing_tokens:
        print(
            f"Missing tokens: {missing_tokens}\nTrying to fetch event for missing tokens..."
        )

    missing_data = []
    for token in missing_tokens:
        missing_data.extend(
            opensea.get_opensea_events(
                contract_address=contract,
                account_address=constants.MINT_ADDRESS,
                event_type="transfer",
                occurred_before=before_time,
                token_id=token,
            )
        )

    df_missing_data = pd.json_normalize(missing_data)

    # Merge missing data with rest of data
    df_all = pd.concat([df, df_missing_data])

    # make sure token_id is an integer
    df_all["asset.token_id"] = df_all["asset.token_id"].astype(int)
    RARITY_DB["TOKEN_ID"] = RARITY_DB["TOKEN_ID"].astype(int)

    # add rarity rank to minting data
    df_all = df_all.merge(RARITY_DB, left_on="asset.token_id", right_on="TOKEN_ID")

    # Keep only the columns we want
    df_all = df_all[
        [
            "transaction.transaction_hash",
            "to_account.address",
            "asset.token_id",
            "asset.owner.address",
            "Rank",
            "transaction.timestamp",
        ]
    ]

    # Rename columns
    df_all.columns = [
        "txid",
        "to_account",
        "TOKEN_ID",
        "current_owner",
        "rank",
        "time",
    ]
    print(f"Downloaded {df_all.shape[0]} events")
    return df_all

"""
Gerenerate Dataset
"""
RARITY_CSV = f"{config.RARITY_FOLDER}/{COLLECTION_NAME}_{METHOD}.csv"
RARITY_DB = pd.read_csv(RARITY_CSV)

mint_db = get_mint_events(CONTRACT, BEFORE_TIME, RARITY_DB)
mint_db = mint_db.sort_values(by=["TOKEN_ID"])
mint_db.to_csv(f"{config.MINTING_FOLDER}/{COLLECTION_NAME}_minting.csv", index=False)

Missing tokens: [1453, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, 1465, 1466, 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1480, 1481, 2375, 2376, 2781, 2782, 2783, 2784, 2785, 2786, 2787, 2788, 2789]
Trying to fetch event for missing tokens...
Downloaded 6000 events