perf: export transactions as csv (#1344)

app/controllers/api/v1/address_transactions_controller.rb

@@ -48,16 +48,7 @@ def show
       def download_csv
         args = params.permit(:id, :start_date, :end_date, :start_number, :end_number, address_transaction: {}).
           merge(address_id: @address.id)
-        data = ExportAddressTransactionsJob.perform_now(args.to_h)
-
-        file =
-          CSV.generate do |csv|
-            csv << [
-              "TXn hash", "Blockno", "UnixTimestamp", "Method", "CKB In", "CKB OUT", "TxnFee(CKB)",
-              "date(UTC)"
-            ]
-            data.each { |row| csv << row }
-          end
+        file = CsvExportable::ExportAddressTransactionsJob.perform_now(args.to_h)
 
         send_data file, type: "text/csv; charset=utf-8; header=present",
                         disposition: "attachment;filename=ckb_transactions.csv"

app/controllers/api/v1/blocks_controller.rb

@@ -52,35 +52,9 @@ def show
       end
 
       def download_csv
-        blocks = Block.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count,
-                              :live_cell_changes, :updated_at)
+        args = params.permit(:start_date, :end_date, :start_number, :end_number, block: {})
+        file = CsvExportable::ExportBlockTransactionsJob.perform_now(args)
 
-        if params[:start_date].present?
-          blocks = blocks.where("timestamp >= ?",
-                                DateTime.strptime(params[:start_date],
-                                                  "%Y-%m-%d").to_time.to_i * 1000)
-        end
-        if params[:end_date].present?
-          blocks = blocks.where("timestamp <= ?",
-                                DateTime.strptime(params[:end_date],
-                                                  "%Y-%m-%d").to_time.to_i * 1000)
-        end
-        blocks = blocks.where("number >= ?", params[:start_number]) if params[:start_number].present?
-        blocks = blocks.where("number <= ?", params[:end_number]) if params[:end_number].present?
-
-        blocks = blocks.order("number desc").limit(5000)
-
-        file =
-          CSV.generate do |csv|
-            csv << ["Blockno", "Transactions", "UnixTimestamp", "Reward(CKB)", "Miner", "date(UTC)"]
-            blocks.find_each.with_index do |block, _index|
-              row = [
-                block.number, block.ckb_transactions_count, (block.timestamp / 1000), block.reward, block.miner_hash,
-                Time.at((block.timestamp / 1000).to_i).in_time_zone("UTC").strftime("%Y-%m-%d %H:%M:%S")
-              ]
-              csv << row
-            end
-          end
         send_data file, type: "text/csv; charset=utf-8; header=present",
                         disposition: "attachment;filename=blocks.csv"
       end

app/controllers/api/v1/udts_controller.rb

@@ -33,17 +33,7 @@ def show
 
       def download_csv
         args = params.permit(:id, :start_date, :end_date, :start_number, :end_number, udt: {})
-        data = ExportUdtTransactionsJob.perform_now(args.to_h)
-
-        file =
-          CSV.generate do |csv|
-            csv << [
-              "Txn hash", "Blockno", "UnixTimestamp", "Method",
-              "Token In", "Token In Name", "Token OUT", "Token OUT Name",
-              "Token From", "Token To", "TxnFee(CKB)", "date(UTC)"
-            ]
-            data.each { |row| csv << row }
-          end
+        file = CsvExportable::ExportUdtTransactionsJob.perform_now(args.to_h)
 
         send_data file, type: "text/csv; charset=utf-8; header=present",
                         disposition: "attachment;filename=udt_transactions.csv"

app/controllers/api/v2/nft/transfers_controller.rb

@@ -3,8 +3,6 @@ module Api
   module V2
     module NFT
       class TransfersController < BaseController
-        before_action :set_token_transfer, only: [:download_csv]
-
         def index
           if params[:collection_id].present?
             if /\A\d+\z/.match?(params[:collection_id])
@@ -46,66 +44,12 @@ def show
         end
 
         def download_csv
-          token_transfers = TokenTransfer.
-            joins(:item, :ckb_transaction).
-            includes(:ckb_transaction, :from, :to).
-            where("token_items.collection_id = ?", @collection.id)
-
-          if params[:start_date].present?
-            token_transfers = token_transfers.where("ckb_transactions.block_timestamp >= ?",
-                                                    DateTime.strptime(params[:start_date],
-                                                                      "%Y-%m-%d").to_time.to_i * 1000)
-          end
-          if params[:end_date].present?
-            token_transfers = token_transfers.where("ckb_transactions.block_timestamp <= ?",
-                                                    DateTime.strptime(params[:end_date],
-                                                                      "%Y-%m-%d").to_time.to_i * 1000)
-          end
-          if params[:start_number].present?
-            token_transfers = token_transfers.where("ckb_transactions.block_number >= ?",
-                                                    params[:start_number])
-          end
-          if params[:end_number].present?
-            token_transfers = token_transfers.where("ckb_transactions.block_number <= ?",
-                                                    params[:end_number])
-          end
+          args = params.permit(:start_date, :end_date, :start_number, :end_number, :collection_id)
+          file = CsvExportable::ExportNFTTransactionsJob.perform_now(args.to_h)
 
-          token_transfers = token_transfers.
-            order("token_transfers.id desc").
-            limit(5000)
-
-          file =
-            CSV.generate do |csv|
-              csv << [
-                "Txn hash", "Blockno", "UnixTimestamp", "NFT ID", "Method", "NFT from", "NFT to", "TxnFee(CKB)",
-                "date(UTC)"
-              ]
-              token_transfers.find_each do |transfer|
-                ckb_transaction = transfer.ckb_transaction
-                row = [
-                  ckb_transaction.tx_hash, ckb_transaction.block_number, ckb_transaction.block_timestamp,
-                  transfer.item.token_id, transfer.action, transfer.from&.address_hash, transfer.to&.address_hash,
-                  ckb_transaction.transaction_fee,
-                  Time.at((ckb_transaction.block_timestamp / 1000).to_i).in_time_zone("UTC").strftime("%Y-%m-%d %H:%M:%S")
-                ]
-                csv << row
-              end
-            end
           send_data file, type: "text/csv; charset=utf-8; header=present",
                           disposition: "attachment;filename=token_transfers.csv"
         end
-
-        private
-
-        def set_token_transfer
-          if params[:collection_id].present?
-            if /\A\d+\z/.match?(params[:collection_id])
-              @collection = TokenCollection.find params[:collection_id]
-            else
-              @collection = TokenCollection.find_by_sn params[:collection_id]
-            end
-          end
-        end
       end
     end
   end

app/jobs/csv_exportable/base_exporter.rb

@@ -0,0 +1,80 @@
+module CsvExportable
+  class BaseExporter < ApplicationJob
+    def perform(*)
+      raise NotImplementedError
+    end
+
+    def generate_csv(header, rows)
+      CSV.generate do |csv|
+        csv << header
+        rows.each { |row| csv << row }
+      end
+    end
+
+    def generate_row(*)
+      raise NotImplementedError
+    end
+
+    def attributes_for_udt_cell(udt_cell)
+      udt_info = Udt.find_by(type_hash: udt_cell.type_hash, published: true)
+      CkbUtils.hash_value_to_s(
+        udt_info: {
+          symbol: udt_info&.symbol,
+          amount: udt_cell.udt_amount,
+          decimal: udt_info&.decimal,
+          uan: udt_info&.uan,
+          type_hash: udt_cell.type_hash
+        }
+      )
+    end
+
+    def capacity_units(cell)
+      units = ["CKB"]
+      if cell[:udt_info]
+        units << (cell[:udt_info][:uan].presence || cell[:udt_info][:symbol])
+      end
+
+      units
+    end
+
+    def cell_capacity(cell, unit)
+      return nil unless cell
+
+      if unit == "CKB"
+        byte = CkbUtils.shannon_to_byte(BigDecimal(cell[:capacity]))
+        return byte.to_s("F")
+      end
+
+      if cell[:udt_info] && cell[:udt_info][:type_hash].present?
+        return parse_udt_amount(cell[:udt_info][:amount], cell[:udt_info][:decimal])
+      end
+    end
+
+    def datetime_utc(timestamp)
+      Time.at((timestamp / 1000).to_i).in_time_zone("UTC").strftime("%Y-%m-%d %H:%M:%S")
+    end
+
+    def parse_transaction_fee(fee)
+      CkbUtils.shannon_to_byte(BigDecimal(fee))
+    end
+
+    def parse_udt_amount(amount, decimal)
+      decimal_int = decimal.to_i
+      amount_big_decimal = BigDecimal(amount)
+      result = amount_big_decimal / (BigDecimal(10)**decimal_int)
+
+      if decimal_int > 20
+        return "#{result.round(20).to_s('F')}..."
+      end
+
+      if result.to_s.length >= 16 || result < BigDecimal("0.000001")
+        return result.round(decimal_int).to_s("F")
+      end
+
+      return result.to_s("F")
+    rescue StandardError => e
+      puts "udt amount parse failed: #{e.message}"
+      return "0"
+    end
+  end
+end

app/jobs/csv_exportable/export_address_transactions_job.rb

@@ -0,0 +1,107 @@
+module CsvExportable
+  class ExportAddressTransactionsJob < BaseExporter
+    def perform(args)
+      tx_ids = AccountBook.joins(:ckb_transaction).
+        where(address_id: args[:address_id]).
+        order(ckb_transaction_id: :asc).
+        limit(5000)
+
+      if args[:start_date].present?
+        start_date = DateTime.strptime(args[:start_date], "%Y-%m-%d").to_time.to_i * 1000
+        tx_ids = tx_ids.where("ckb_transactions.block_timestamp >= ?", start_date)
+      end
+
+      if args[:end_date].present?
+        end_date = DateTime.strptime(args[:end_date], "%Y-%m-%d").to_time.to_i * 1000
+        tx_ids = tx_ids.where("ckb_transactions.block_timestamp <= ?", end_date)
+      end
+
+      if args[:start_number].present?
+        tx_ids = tx_ids.where("ckb_transactions.block_number >= ?", args[:start_number])
+      end
+
+      if args[:end_number].present?
+        tx_ids = tx_ids.where("ckb_transactions.block_number <= ?", args[:end_number])
+      end
+
+      ckb_transactions = CkbTransaction.includes(:inputs, :outputs).
+        select(:id, :tx_hash, :transaction_fee, :block_id, :block_number, :block_timestamp, :updated_at).
+        where(id: tx_ids.pluck(:ckb_transaction_id))
+
+      rows = []
+      ckb_transactions.find_in_batches(batch_size: 1000, order: :desc) do |transactions|
+        transactions.each do |transaction|
+          row = generate_row(transaction, args[:address_id])
+          next if row.blank?
+
+          rows += row
+        end
+      end
+
+      header = [
+        "Txn hash", "Blockno", "UnixTimestamp", "Token", "Method", "Token In",
+        "Token Out", "Token Balance Change", "TxnFee(CKB)", "date(UTC)"
+      ]
+
+      generate_csv(header, rows)
+    end
+
+    def generate_row(transaction, address_id)
+      inputs = simple_display_inputs(transaction, address_id)
+      outputs = simple_display_outputs(transaction, address_id)
+      datetime = datetime_utc(transaction.block_timestamp)
+
+      rows = []
+      max = [inputs.size, outputs.size].max
+      (0..max - 1).each do |i|
+        units = capacity_units(outputs[i] || inputs[i])
+        units.each do |unit|
+          token_in = cell_capacity(inputs[i], unit)
+          token_out = cell_capacity(outputs[i], unit)
+          balance_change = token_out.to_f - token_in.to_f
+          method = balance_change.positive? ? "PAYMENT RECEIVED" : "PAYMENT SENT"
+          fee = parse_transaction_fee(transaction.transaction_fee)
+
+          rows << [
+            transaction.tx_hash,
+            transaction.block_number,
+            transaction.block_timestamp,
+            unit,
+            method,
+            (token_in || "/"),
+            (token_out || "/"),
+            balance_change,
+            (unit == "CKB" ? fee : "/"),
+            datetime
+          ]
+        end
+      end
+
+      rows
+    end
+
+    def simple_display_inputs(transaction, address_id)
+      previous_cell_outputs = transaction.inputs.where(address_id: address_id).order(id: :asc)
+      previous_cell_outputs.map do |cell_output|
+        display_input = { capacity: cell_output.capacity }
+        if cell_output.udt?
+          display_input.merge!(attributes_for_udt_cell(cell_output))
+        end
+
+        CkbUtils.hash_value_to_s(display_input)
+      end
+    end
+
+    def simple_display_outputs(transaction, address_id)
+      cell_outputs = transaction.outputs.where(address_id: address_id).order(id: :asc)
+      cell_outputs.map do |cell_output|
+        display_output = { capacity: cell_output.capacity }
+        if cell_output.udt?
+          display_output.merge!(attributes_for_udt_cell(cell_output))
+        end
+
+        CkbUtils.hash_value_to_s(display_output)
+      end
+    end
+  end
+end

app/jobs/csv_exportable/export_block_transactions_job.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module CsvExportable
+  class ExportBlockTransactionsJob < BaseExporter
+    def perform(args)
+      blocks = Block.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count,
+                            :live_cell_changes, :updated_at)
+
+      if args[:start_date].present?
+        start_date = DateTime.strptime(args[:start_date], "%Y-%m-%d").to_time.to_i * 1000
+        blocks = blocks.where("timestamp >= ?", start_date)
+      end
+
+      if args[:end_date].present?
+        end_date = DateTime.strptime(args[:end_date], "%Y-%m-%d").to_time.to_i * 1000
+        blocks = blocks.where("timestamp <= ?", end_date)
+      end
+
+      blocks = blocks.where("number >= ?", args[:start_number]) if args[:start_number].present?
+      blocks = blocks.where("number <= ?", args[:end_number]) if args[:end_number].present?
+      blocks = blocks.order("number desc").limit(5000)
+
+      rows = []
+      blocks.find_in_batches(batch_size: 10, order: :desc) do |blocks|
+        blocks.each do |block|
+          row = generate_row(block)
+          next if row.blank?
+
+          rows << row
+        end
+      end
+
+      header = ["Blockno", "Transactions", "UnixTimestamp", "Reward(CKB)", "Miner", "date(UTC)"]
+
+      generate_csv(header, rows)
+    end
+
+    def generate_row(block)
+      datetime = datetime_utc(block.timestamp)
+      reward = CkbUtils.shannon_to_byte(BigDecimal(block.reward))
+
+      [
+        block.number,
+        block.ckb_transactions_count,
+        block.timestamp,
+        reward,
+        block.miner_hash,
+        datetime
+      ]
+    end
+  end
+end