[feat] data layer

.gitignore

@@ -65,3 +65,6 @@ src/data/crowdin/bucketsAwaitingReviewReport.csv
 build-storybook.log
 build-archive.log
 storybook-static
+
+# Trigger
+.trigger

package.json

@@ -27,13 +27,15 @@
     "test:e2e": "playwright test",
     "test:e2e:ui": "playwright test --ui",
     "test:e2e:debug": "playwright test --debug",
-    "test:e2e:report": "playwright show-report tests/e2e/__report__"
+    "test:e2e:report": "playwright show-report tests/e2e/__report__",
+    "trigger:dev": "dotenv -e .env -- npx trigger.dev@latest dev"
   },
   "dependencies": {
     "@aws-sdk/client-ses": "^3.859.0",
     "@crowdin/crowdin-api-client": "^1.25.0",
     "@docsearch/react": "^3.5.2",
     "@hookform/resolvers": "^3.8.0",
+    "@netlify/blobs": "^10.4.1",
     "@next/bundle-analyzer": "^14.2.5",
     "@radix-ui/react-accordion": "^1.2.0",
     "@radix-ui/react-avatar": "^1.1.2",
@@ -61,6 +63,7 @@
     "@tanstack/react-query": "^5.66.7",
     "@tanstack/react-table": "^8.19.3",
     "@tanstack/react-virtual": "^3.13.12",
+    "@trigger.dev/sdk": "^4.1.2",
     "@types/canvas-confetti": "^1.9.0",
     "@types/three": "^0.177.0",
     "@wagmi/core": "^2.17.3",
@@ -138,6 +141,7 @@
     "chromatic": "12.0.0",
     "decompress": "^4.2.1",
     "dotenv": "^16.5.0",
+    "dotenv-cli": "^11.0.0",
     "eslint": "^8.57.1",
     "eslint-config-next": "^14.2.2",
     "eslint-config-prettier": "^9",

src/data-layer/api/fetchApps.ts

@@ -0,0 +1,192 @@
+import { AppCategoryEnum, AppData } from "@/lib/types"
+
+export const FETCH_APPS_TASK_ID = "fetch-apps"
+
+/**
+ * Fetch apps data from Google Sheets.
+ * Returns the fetched apps data organized by category.
+ */
+export async function fetchApps(): Promise<Record<string, AppData[]>> {
+  const googleApiKey = process.env.GOOGLE_API_KEY
+  const sheetId = process.env.GOOGLE_SHEET_ID_DAPPS
+
+  if (!sheetId) {
+    throw new Error("Google Sheets ID not set")
+  }
+
+  if (!googleApiKey) {
+    throw new Error("Google API key not set")
+  }
+
+  console.log("Starting apps data fetch from Google Sheets")
+
+  // First, get the spreadsheet metadata to see what sheets exist
+  const metadataUrl = `https://sheets.googleapis.com/v4/spreadsheets/${sheetId}?key=${googleApiKey}`
+
+  const metadataResponse = await fetch(metadataUrl)
+
+  if (!metadataResponse.ok) {
+    const errorText = await metadataResponse.text()
+    console.error("Metadata fetch error", {
+      status: metadataResponse.status,
+      statusText: metadataResponse.statusText,
+      error: errorText,
+    })
+    throw new Error(
+      `Metadata fetch failed: ${metadataResponse.status} ${metadataResponse.statusText}`
+    )
+  }
+
+  const metadata = await metadataResponse.json()
+  const sheetNames =
+    metadata.sheets?.map(
+      (sheet: { properties: { title: string } }) => sheet.properties.title
+    ) || []
+
+  // Filter out sheets that are not valid AppCategoryEnum values
+  const appCategorySheetNames = sheetNames.filter((name: string) =>
+    Object.values(AppCategoryEnum).includes(name as AppCategoryEnum)
+  )
+
+  console.log(`Found ${appCategorySheetNames.length} app category sheets`)
+
+  const appsOfTheWeek = await fetch(
+    `https://sheets.googleapis.com/v4/spreadsheets/${sheetId}/values/App%20of%20the%20day!A2:C?majorDimension=ROWS&key=${googleApiKey}`
+  )
+
+  if (!appsOfTheWeek.ok) {
+    console.warn(
+      `Failed to fetch from sheet Apps of the day: ${appsOfTheWeek.status} ${appsOfTheWeek.statusText}`
+    )
+  }
+
+  const appsOfTheWeekData = await appsOfTheWeek.json()
+
+  const result: Record<string, AppData[]> = {}
+
+  // Fetch and process data from each sheet
+  for (const sheetName of appCategorySheetNames) {
+    const dataUrl = `https://sheets.googleapis.com/v4/spreadsheets/${sheetId}/values/${sheetName}!A:Z?majorDimension=ROWS&key=${googleApiKey}`
+    const dataResponse = await fetch(dataUrl)
+
+    if (!dataResponse.ok) {
+      console.warn(
+        `Failed to fetch from sheet ${sheetName}: ${dataResponse.status} ${dataResponse.statusText}`
+      )
+      result[sheetName] = []
+      continue
+    }
+
+    const data = await dataResponse.json()
+    const rows = data.values || []
+
+    if (rows.length === 0) {
+      result[sheetName] = []
+      continue
+    }
+
+    // Process data rows (skip header)
+    const dataRows = rows.slice(1).filter((row: string[]) => {
+      // Filter out completely empty rows or rows without a name
+      return row.length > 0 && row[0]?.trim() !== ""
+    })
+
+    const apps: AppData[] = dataRows
+      .map((row: string[]) => {
+        // Map row data to app object
+        const appData = {
+          name: row[0] || "",
+          url: row[1] || "",
+          description: row[2] || "",
+          image: row[3] || "", // Use the SVG data directly from the Logo Image column
+          category: getCategoryFromSheetName(sheetName),
+          subCategory: parseCommaSeparated(row[5] || ""),
+          networks: parseCommaSeparated(row[6] || ""),
+          screenshots: parseCommaSeparated(row[7] || ""),
+          bannerImage: row[8] || "",
+          platforms: parseCommaSeparated(row[9] || ""),
+          twitter: row[10] || "",
+          github: row[11] || "",
+          discord: row[12] || "",
+          kpiUrl: row[13] || "",
+          sortingWeight: parseInt(row[14] || "0") || 0,
+          discover: row[15]?.toLowerCase() === "true",
+          highlight: row[16]?.toLowerCase() === "true",
+          languages: parseCommaSeparated(row[17] || ""),
+          parentCompany: row[18] || "",
+          parentCompanyURL: row[19] || "",
+          openSource: row[20]?.toLowerCase() === "true",
+          contractAddress: row[21] || "",
+          dateOfLaunch: row[22] || "",
+          lastUpdated: row[23] || "",
+          ready: row[24]?.toLowerCase(),
+          devconnect: row[25]?.toLowerCase(),
+          ...parseAppOfTheWeekDate(row[0], appsOfTheWeekData),
+        }
+
+        return appData as unknown as AppData
+      })
+      .filter((app: AppData) => app.name && app.url) // Filter out apps without name or URL
+      .filter((app: AppData) => app.ready === "true")
+
+    result[sheetName] = apps
+    console.log(`Processed ${apps.length} apps from ${sheetName}`)
+  }
+
+  const totalApps = Object.values(result).reduce(
+    (sum, apps) => sum + apps.length,
+    0
+  )
+
+  console.log(
+    `Successfully fetched ${totalApps} apps across ${Object.keys(result).length} categories`
+  )
+
+  return result
+}
+
+// Helper function to map sheet names to AppCategoryEnum
+function getCategoryFromSheetName(sheetName: string): AppCategoryEnum {
+  switch (sheetName) {
+    case "DeFi":
+      return AppCategoryEnum.DEFI
+    case "Collectibles":
+      return AppCategoryEnum.COLLECTIBLE
+    case "Social":
+      return AppCategoryEnum.SOCIAL
+    case "Gaming":
+      return AppCategoryEnum.GAMING
+    case "Bridge":
+      return AppCategoryEnum.BRIDGE
+    case "Productivity":
+      return AppCategoryEnum.PRODUCTIVITY
+    case "Privacy":
+      return AppCategoryEnum.PRIVACY
+    case "DAO":
+      return AppCategoryEnum.GOVERNANCE_DAO
+    default:
+      return AppCategoryEnum.DEFI // Default fallback
+  }
+}
+
+// Helper function to parse comma-separated strings into arrays
+function parseCommaSeparated(value: string): string[] {
+  if (!value || value.trim() === "") return []
+  return value
+    .split(",")
+    .map((item) => item.trim())
+    .filter(Boolean)
+}
+
+const parseAppOfTheWeekDate = (
+  appName: string,
+  appsOfTheWeekData: { values: Array<[string, string, string]> }
+): { appOfTheWeekStartDate: Date | null; appOfTheWeekEndDate: Date | null } => {
+  const appOfTheWeek = appsOfTheWeekData.values.find(
+    (app: [string, string, string]) => app[0] === appName
+  )
+  return {
+    appOfTheWeekStartDate: appOfTheWeek ? new Date(appOfTheWeek[1]) : null,
+    appOfTheWeekEndDate: appOfTheWeek ? new Date(appOfTheWeek[2]) : null,
+  }
+}

src/data-layer/api/fetchBeaconChainEpoch.ts

@@ -0,0 +1,40 @@
+import type { BeaconchainEpochData, EpochResponse } from "@/lib/types"
+
+export const FETCH_BEACONCHAIN_EPOCH_TASK_ID = "fetch-beaconchain-epoch"
+
+/**
+ * Fetch beaconchain epoch data from Beaconcha.in API.
+ * Returns the latest epoch data including total ETH staked and validator count.
+ */
+export async function fetchBeaconChainEpoch(): Promise<BeaconchainEpochData> {
+  const base = "https://beaconcha.in"
+  const endpoint = "api/v1/epoch/latest"
+  const { href } = new URL(endpoint, base)
+
+  console.log("Starting beaconchain epoch data fetch")
+
+  const response = await fetch(href)
+
+  if (!response.ok) {
+    const status = response.status
+    console.warn("Beaconcha.in fetch non-OK", { status, url: href })
+    const error = `Beaconcha.in responded with status ${status}`
+    throw new Error(error)
+  }
+
+  const json: EpochResponse = await response.json()
+  const { validatorscount, eligibleether } = json.data
+  const totalEthStaked = Math.floor(eligibleether * 1e-9) // `eligibleether` value returned in `gwei`
+  const timestamp = Date.now()
+
+  console.log("Successfully fetched beaconchain epoch data", {
+    totalEthStaked,
+    validatorscount,
+    timestamp,
+  })
+
+  return {
+    totalEthStaked: { value: totalEthStaked, timestamp },
+    validatorscount: { value: validatorscount, timestamp },
+  }
+}

src/data-layer/api/fetchBeaconChainEthstore.ts

@@ -0,0 +1,35 @@
+import type { EthStoreResponse, MetricReturnData } from "@/lib/types"
+
+export const FETCH_BEACONCHAIN_ETHSTORE_TASK_ID = "fetch-beaconchain-ethstore"
+
+/**
+ * Fetch beaconchain ethstore data from Beaconcha.in API.
+ * Returns the latest APR data.
+ */
+export async function fetchBeaconChainEthstore(): Promise<MetricReturnData> {
+  const base = "https://beaconcha.in"
+  const endpoint = "api/v1/ethstore/latest"
+  const { href } = new URL(endpoint, base)
+
+  console.log("Starting beaconchain ethstore data fetch")
+
+  const response = await fetch(href)
+
+  if (!response.ok) {
+    const status = response.status
+    console.warn("Beaconcha.in fetch non-OK", { status, url: href })
+    const error = `Beaconcha.in responded with status ${status}`
+    throw new Error(error)
+  }
+
+  const json: EthStoreResponse = await response.json()
+  const apr = json.data.apr
+  const timestamp = Date.now()
+
+  console.log("Successfully fetched beaconchain ethstore data", {
+    apr,
+    timestamp,
+  })
+
+  return { value: apr, timestamp }
+}

src/data-layer/api/fetchBlobscanStats.ts

@@ -0,0 +1,51 @@
+export const FETCH_BLOBSCAN_STATS_TASK_ID = "fetch-blobscan-stats"
+
+type BlobscanOverallStats = {
+  avgBlobAsCalldataFee: number
+  avgBlobFee: number
+  avgBlobGasPrice: number
+  avgMaxBlobGasFee: number
+  totalBlobGasUsed: string
+  totalBlobAsCalldataGasUsed: string
+  totalBlobFee: string
+  totalBlobAsCalldataFee: string
+  totalBlobs: number
+  totalBlobSize: string
+  totalBlocks: number
+  totalTransactions: number
+  totalUniqueBlobs: number
+  totalUniqueReceivers: number
+  totalUniqueSenders: number
+  updatedAt: string
+}
+
+/**
+ * Fetch the overall stats from Blobscan
+ *
+ * @see https://api.blobscan.com/#/stats/stats-getOverallStats
+ */
+export async function fetchBlobscanStats(): Promise<BlobscanOverallStats> {
+  const url = "https://api.blobscan.com/stats/overall"
+
+  console.log("Starting blobscan stats data fetch")
+
+  const response = await fetch(url)
+
+  if (!response.ok) {
+    const status = response.status
+    console.warn("Blobscan fetch non-OK", { status, url })
+    const error = `Blobscan responded with status ${status}`
+    throw new Error(error)
+  }
+
+  const json: [BlobscanOverallStats] = await response.json()
+  const stats = json[0]
+
+  console.log("Successfully fetched blobscan stats data", {
+    totalBlobs: stats.totalBlobs,
+    totalTransactions: stats.totalTransactions,
+    updatedAt: stats.updatedAt,
+  })
+
+  return stats
+}