WIP: Ethereum Sync Support

packages/bitcore-node/package.json

@@ -60,6 +60,7 @@
     "mocha": "^5.0.5",
     "sinon": "^4.4.8",
     "supertest": "^3.0.0",
+    "types-bn": "0.0.1",
     "typescript": "^2.8.3",
     "typescript-eslint-parser": "^15.0.0"
   },
@@ -72,13 +73,17 @@
     "bitcore-p2p-cash": "^8.0.0",
     "body-parser": "^1.18.3",
     "cors": "^2.8.4",
+    "ethereumjs-block": "^2.0.1",
+    "ethereumjs-devp2p": "^2.5.0",
+    "ethereumjs-tx": "^1.3.7",
     "express": "^4.16.3",
     "lru-cache": "^4.1.3",
     "mkdirp": "^0.5.1",
     "mongodb": "^3.1.1",
     "mongoose": "^5.0.11",
     "progress": "^2.0.0",
     "request": "^2.86.0",
+    "rlp-encoding": "^3.0.0",
     "rxjs": "^6.1.0",
     "secp256k1": "^3.5.0",
     "snappy": "^6.0.1",

packages/bitcore-node/src/chain.ts

@@ -1,4 +1,4 @@
-module.exports = {
+export const Chain = {
   BTC: {
     lib: require('bitcore-lib'),
     p2p: require('bitcore-p2p'),

packages/bitcore-node/src/models/base.ts

@@ -1,7 +1,7 @@
 import { Storage } from '../services/storage';
 import {  ObjectID, Collection, MongoClient, Db } from 'mongodb';
 
-export type MongoBound<T> = T & {_id: ObjectID};
+export type MongoBound<T> = T & { _id: ObjectID };
 export abstract class BaseModel<T> {
   connected = false;
   client?: MongoClient;

packages/bitcore-node/src/models/block.ts

@@ -2,9 +2,10 @@ import { CoinModel, SpentHeightIndicators } from './coin';
 import { TransactionModel } from './transaction';
 import { TransformOptions } from '../types/TransformOptions';
 import { LoggifyClass } from '../decorators/Loggify';
-import { Bitcoin } from '../types/namespaces/Bitcoin';
 import { BaseModel, MongoBound } from './base';
 import logger from '../logger';
+import { Bucket } from '../types/namespaces/ChainAdapter';
+import { VerboseTransaction } from '../services/p2p';
 
 export type IBlock = {
   chain: string;
@@ -46,16 +47,21 @@ export class Block extends BaseModel<IBlock> {
   }
 
   async addBlock(params: {
-    block: any;
+    block: Bucket<IBlock>;
+    transactions: Array<Bucket<VerboseTransaction>>;
     parentChain?: string;
     forkHeight?: number;
     initialSyncComplete: boolean;
     chain: string;
     network: string;
   }) {
-    const { block, chain, network, parentChain, forkHeight, initialSyncComplete } = params;
-    const header = block.header.toObject();
-    const blockTime = header.time * 1000;
+    const { block, transactions, chain, network, parentChain, forkHeight, initialSyncComplete } = params;
+    const header = {
+      prevHash: block.previousBlockHash,
+      hash: block.hash,
+      time: block.time
+    };
+    const blockTime = block.time.getTime();
 
     const reorg = await this.handleReorg({ header, chain, network });
 
@@ -85,16 +91,17 @@ export class Block extends BaseModel<IBlock> {
           network,
           hash: block.hash,
           height,
-          version: header.version,
+          version: block.version,
           previousBlockHash: header.prevHash,
-          merkleRoot: header.merkleRoot,
+          merkleRoot: block.merkleRoot,
           time: new Date(blockTime),
           timeNormalized: new Date(blockTimeNormalized),
-          bits: header.bits,
-          nonce: header.nonce,
-          transactionCount: block.transactions.length,
-          size: block.toBuffer().length,
-          reward: block.transactions[0].outputAmount
+          bits: block.bits,
+          nonce: block.nonce,
+          transactionCount: transactions.length,
+          size: block.size,
+          reward: block.reward,
+          ...block.bucket
         }
       },
       { upsert: true }
@@ -109,7 +116,7 @@ export class Block extends BaseModel<IBlock> {
     }
 
     await TransactionModel.batchImport({
-      txs: block.transactions,
+      txs: transactions,
       blockHash: header.hash,
       blockTime: new Date(blockTime),
       blockTimeNormalized: new Date(blockTimeNormalized),
@@ -134,7 +141,11 @@ export class Block extends BaseModel<IBlock> {
     return BlockModel.collection.findOne({ chain, network, processed: true }, { sort: { height: -1 } });
   }
 
-  async handleReorg(params: { header?: Bitcoin.Block.HeaderObj; chain: string; network: string }): Promise<boolean> {
+  async handleReorg(params: {
+    header?: { hash: string; prevHash: string };
+    chain: string;
+    network: string;
+  }): Promise<boolean> {
     const { header, chain, network } = params;
     const localTip = await this.getLocalTip(params);
     if (header && localTip && localTip.hash === header.prevHash) {

packages/bitcore-node/src/models/transaction.ts

@@ -1,17 +1,16 @@
 import { CoinModel, ICoin, SpentHeightIndicators } from './coin';
 import { WalletAddressModel } from './walletAddress';
 import { partition } from '../utils/partition';
-import { ObjectID } from 'bson';
+import { ObjectID } from 'mongodb';
 import { TransformOptions } from '../types/TransformOptions';
 import { LoggifyClass } from '../decorators/Loggify';
-import { Bitcoin } from '../types/namespaces/Bitcoin';
 import { BaseModel, MongoBound } from './base';
 import logger from '../logger';
 import config from '../config';
 import { BulkWriteOpResultObject } from 'mongodb';
 import { StreamingFindOptions, Storage } from '../services/storage';
-
-const Chain = require('../chain');
+import { VerboseTransaction } from "../services/p2p";
+import { Bucket } from "../types/namespaces/ChainAdapter";
 
 export type ITransaction = {
   txid: string;
@@ -21,10 +20,8 @@ export type ITransaction = {
   blockHash?: string;
   blockTime?: Date;
   blockTimeNormalized?: Date;
-  coinbase: boolean;
   fee: number;
   size: number;
-  locktime: number;
   wallets: ObjectID[];
 };
 
@@ -45,7 +42,7 @@ export class Transaction extends BaseModel<ITransaction> {
   }
 
   async batchImport(params: {
-    txs: Array<Bitcoin.Transaction>;
+    txs: Array<Bucket<VerboseTransaction>>;
     height: number;
     mempoolTime?: Date;
     blockTime?: Date;
@@ -91,7 +88,7 @@ export class Transaction extends BaseModel<ITransaction> {
   }
 
   async addTransactions(params: {
-    txs: Array<Bitcoin.Transaction>;
+    txs: Array<Bucket<VerboseTransaction>>;
     height: number;
     blockTime?: Date;
     blockHash?: string;
@@ -104,7 +101,7 @@ export class Transaction extends BaseModel<ITransaction> {
     mintOps?: Array<any>;
   }) {
     let { blockHash, blockTime, blockTimeNormalized, chain, height, network, txs, initialSyncComplete } = params;
-    let txids = txs.map(tx => tx._hash);
+    let txids = txs.map(tx => tx.txid);
 
     type TaggedCoin = ICoin & { _id: string };
     let mintWallets;
@@ -151,10 +148,9 @@ export class Transaction extends BaseModel<ITransaction> {
               blockHash,
               blockTime,
               blockTimeNormalized,
-              coinbase: tx.isCoinbase(),
-              size: tx.toBuffer().length,
-              locktime: tx.nLockTime,
-              wallets
+              size: tx.size,
+              wallets,
+              ...tx.bucket
             }
           },
           upsert: true,
@@ -166,7 +162,7 @@ export class Transaction extends BaseModel<ITransaction> {
   }
 
   async getMintOps(params: {
-    txs: Array<Bitcoin.Transaction>;
+    txs: Array<VerboseTransaction>;
     height: number;
     parentChain?: string;
     forkHeight?: number;
@@ -189,25 +185,15 @@ export class Transaction extends BaseModel<ITransaction> {
         .toArray();
     }
     for (let tx of txs) {
-      tx._hash = tx.hash;
-      let txid = tx._hash;
-      let isCoinbase = tx.isCoinbase();
+      let txid = tx.txid;
       for (let [index, output] of tx.outputs.entries()) {
         let parentChainCoin = parentChainCoins.find(
           (parentChainCoin: ICoin) => parentChainCoin.mintTxid === txid && parentChainCoin.mintIndex === index
         );
         if (parentChainCoin) {
           continue;
         }
-        let address = '';
-        let scriptBuffer = output.script && output.script.toBuffer();
-        if (scriptBuffer) {
-          address = output.script.toAddress(network).toString(true);
-          if (address === 'false' && output.script.classify() === 'Pay to public key') {
-            let hash = Chain[chain].lib.crypto.Hash.sha256ripemd160(output.script.chunks[0].buf);
-            address = Chain[chain].lib.Address(hash, network).toString(true);
-          }
-        }
+        let address = output.address;
 
         mintOps.push({
           updateOne: {
@@ -223,12 +209,11 @@ export class Transaction extends BaseModel<ITransaction> {
                 chain,
                 network,
                 mintHeight: height,
-                coinbase: isCoinbase,
-                value: output.satoshis,
+                value: output.value,
                 address,
-                script: scriptBuffer,
                 spentHeight: SpentHeightIndicators.unspent,
-                wallets: []
+                wallets: [],
+                ...output.bucket
               }
             },
             upsert: true,
@@ -258,7 +243,7 @@ export class Transaction extends BaseModel<ITransaction> {
   }
 
   getSpendOps(params: {
-    txs: Array<Bitcoin.Transaction>;
+    txs: Array<Bucket<VerboseTransaction>>;
     height: number;
     parentChain?: string;
     forkHeight?: number;
@@ -277,13 +262,13 @@ export class Transaction extends BaseModel<ITransaction> {
       mintMap[mintOp.updateOne.filter.mintTxid][mintOp.updateOne.filter.mintIndex] = mintOp;
     }
     for (let tx of txs) {
-      if (tx.isCoinbase()) {
+      if (tx.bucket.coinbase) {
         continue;
       }
-      let txid = tx._hash;
+      let txid = tx.blockHash;
       for (let input of tx.inputs) {
-        let inputObj = input.toObject();
-        let sameBlockSpend = mintMap[inputObj.prevTxId] && mintMap[inputObj.prevTxId][inputObj.outputIndex];
+        let inputObj = input;
+        let sameBlockSpend = mintMap[inputObj.mintTxid] && mintMap[inputObj.mintTxid][inputObj.mintIndex];
         if (sameBlockSpend) {
           sameBlockSpend.updateOne.update.$set.spentHeight = height;
           sameBlockSpend.updateOne.update.$set.spentTxid = txid;
@@ -295,8 +280,8 @@ export class Transaction extends BaseModel<ITransaction> {
         const updateQuery: any = {
           updateOne: {
             filter: {
-              mintTxid: inputObj.prevTxId,
-              mintIndex: inputObj.outputIndex,
+              mintTxid: inputObj.mintTxid,
+              mintIndex: inputObj.mintIndex,
               spentHeight: { $lt: SpentHeightIndicators.minimum },
               chain,
               network
@@ -321,23 +306,10 @@ export class Transaction extends BaseModel<ITransaction> {
   }
 
   _apiTransform(tx: Partial<MongoBound<ITransaction>>, options: TransformOptions): Partial<ITransaction> | string {
-    let transform = {
-      _id: tx._id,
-      txid: tx.txid,
-      network: tx.network,
-      blockHeight: tx.blockHeight,
-      blockHash: tx.blockHash,
-      blockTime: tx.blockTime,
-      blockTimeNormalized: tx.blockTimeNormalized,
-      coinbase: tx.coinbase,
-      locktime: tx.locktime,
-      size: tx.size,
-      fee: tx.fee
-    };
     if (options && options.object) {
-      return transform;
+      return tx;
     }
-    return JSON.stringify(transform);
+    return JSON.stringify(tx);
   }
 }
 export let TransactionModel = new Transaction();

packages/bitcore-node/src/providers/chain-state/index.ts

@@ -1,14 +1,14 @@
 import { BTCStateProvider } from './btc/btc';
 import { BCHStateProvider } from './bch/bch';
-import { ETHStateProvider } from './eth/eth';
 import { BATStateProvider } from './erc20/tokens/bat';
 import { CSP } from '../../types/namespaces/ChainStateProvider';
 import { Chain } from '../../types/ChainNetwork';
+import { InternalStateProvider } from "./internal/internal";
 
 const services: CSP.ChainStateServices = {
   BTC: new BTCStateProvider(),
   BCH: new BCHStateProvider(),
-  ETH: new ETHStateProvider(),
+  ETH: new InternalStateProvider('ETH'),
   BAT: new BATStateProvider()
 };
 

packages/bitcore-node/src/providers/chain-state/internal/internal.ts

@@ -75,7 +75,7 @@ export class InternalStateProvider implements CSP.IChainStateService {
   streamBlocks(params: CSP.StreamBlocksParams) {
     const { stream } = params;
     const { query, options } = this.getBlocksQuery(params);
-    Storage.apiStreamingFind(BlockModel, query, options, stream);
+    return Storage.apiStreamingFind(BlockModel, query, options, stream);
   }
 
   async getBlocks(params: CSP.GetBlockParams) {

packages/bitcore-node/src/routes/api/block.ts

@@ -22,7 +22,7 @@ router.get('/', async function(req: Request, res: Response) {
 router.get('/tip', async function(req: Request, res: Response) {
   let { chain, network } = req.params;
   try {
-    let tip = await ChainStateProvider.getBlock({ chain, network });
+    let tip = await ChainStateProvider.getLocalTip({ chain, network });
     return res.json(tip);
   } catch (err) {
     return res.status(500).send(err);

packages/bitcore-node/src/server.ts

@@ -1,4 +1,4 @@
-import { P2pService } from './services/p2p';
+import { P2P } from './services/p2p';
 import { Storage } from './services/storage';
 import { Worker } from './services/worker';
 import { Api } from './services/api';
@@ -9,31 +9,31 @@ let args = parseArgv([], ['DEBUG']);
 const startServices = async () => {
   await Storage.start({});
   await Worker.start();
-  P2pService.startConfiguredChains();
+  P2P.startConfiguredChains();
 };
 
-const runMaster = async() => {
+const runMaster = async () => {
   await startServices();
   // start the API on master if we are in debug
-  if(args.DEBUG){
+  if (args.DEBUG) {
     Api.start();
   }
 };
 
-const runWorker = async() => {
+const runWorker = async () => {
   // don't run any workers when in debug mode
-  if(!args.DEBUG){
+  if (!args.DEBUG) {
     // Api will automatically start storage if it isn't already running
     Api.start();
   }
-}
+};
 
-const start = async() => {
-  if(cluster.isMaster){
+const start = async () => {
+  if (cluster.isMaster) {
     await runMaster();
-  } else{
+  } else {
     await runWorker();
   }
-}
+};
 
 start();