wip: 🚧 downloading of files

readme.md

@@ -19,3 +19,7 @@ https://elibrary.ferc.gov/eLibrarywebapi/api/Docket/getDocketDescription/ER03-91
 } -->
 
 <!-- https://elibrary.ferc.gov/eLibrarywebapi/api/Docket/GetATMSdocs/rbCreateDate/05-15-2024/05-15-2024/DocketFullNumber -->
+
+<!-- https://elibrary.ferc.gov/eLibrarywebapi/api/File/DownloadPDF?accesssionNumber=20221213-4015 -->
+
+<!-- https://elibrary.ferc.gov/eLibrarywebapi/api/File/DownloadP8File -->

src/search/general.ts

@@ -1,4 +1,6 @@
 import type { GeneralSearchParams, GeneralSearchResult } from "~/types";
+import { Search } from "./search";
+import { fetchFunction } from "~/util/fetch";
 
 const defaultSearch: GeneralSearchParams = {
   searchText: '*',
@@ -18,7 +20,7 @@ const defaultSearch: GeneralSearchParams = {
     },
   ],
   resultsPerPage: 100,
-  curPage: 0,
+  curPage: 1,
   classTypes: [],
   sortBy: '',
   groupBy: 'NONE',
@@ -27,32 +29,41 @@ const defaultSearch: GeneralSearchParams = {
 }
 
 
-export default class GeneralSearch {
+export default class GeneralSearch extends Search<GeneralSearchParams, GeneralSearchResult> {
+  readonly defaultParams = defaultSearch
+  readonly fetch = fetchFunction<GeneralSearchParams, GeneralSearchResult>("Search/AdvancedSearch")
 
-  search: GeneralSearchParams
-  data: GeneralSearchResult | undefined
+  constructor(params: Partial<GeneralSearchParams>) {
+    super()
+    this.params = { ...this.defaultParams, ...params }
+    if (this.params.curPage === 0) this.params.curPage = 1
+  }
 
+  async getData() {
+    this.data = await this.fetch(this.params)
+  }
 
-  constructor(search: Partial<GeneralSearchParams>){
-    this.search = {...defaultSearch, ...search}
+  updateParams(params: Partial<GeneralSearch>) {
+    this.params = { ...defaultSearch, ...params }
   }
 
-  async fetch() {
-    const r = await fetch('https://elibrary.ferc.gov/eLibrarywebapi/api/Search/AdvancedSearch', {
-      headers: {
-        'content-type': 'application/json',
-      },
-      body: JSON.stringify(this.search),
-      method: 'post',
+  async nextPage(): Promise<boolean> {
+    const { resultsPerPage, curPage } = this.params
+    if ((this.data?.totalHits ?? -1) > (resultsPerPage * (curPage))) {
+      this.params.curPage++
+      await this.getData()
+      return true
     }
-  )
-  this.data = await (r).json()
+    return false
   }
 
-
-
-  updateSearch(){}
-  nextPage() {}
-  prevPage() {}
-  page() {}
+  async prevPage(): Promise<boolean> {
+    const { curPage } = this.params
+    if (curPage > 1) {
+      this.params.curPage--
+      await this.getData()
+      return true
+    }
+    return false
+  }
 }

src/search/search.ts

@@ -0,0 +1,10 @@
+export abstract class Search<
+  SearchParams extends Record<string, unknown>,
+  SearchResults extends Record<string, unknown>
+> {
+  params: SearchParams = {} as SearchParams
+  abstract defaultParams: SearchParams
+
+  data: SearchResults | undefined
+  abstract readonly fetch: (params: SearchParams) => Promise<SearchResults>
+}

src/types/DownloadFile.ts

@@ -0,0 +1,8 @@
+export type DownloadFileParams = {
+  FileType: string
+  accession: string
+  fileid: number
+  FileIDAll: string
+  fileidLst: string[]
+  Islegacy: boolean
+}

src/types/GeneralSearch.ts

@@ -7,8 +7,8 @@ import {
 } from './stubs'
 
 export type DocketSearch = {
-  docketNumber: string
-  subDocketNumbers: string[]
+  docketNumber: string | null
+  subDocketNumbers: string[] | null
 }
 export type DateSearch = {
   dateType: 'issued_date' | 'filed_date' | 'posted_date'
@@ -75,7 +75,7 @@ export type GeneralSearchResult = {
   searchHits: {
     reference: string
     documentId: string
-    acesssionNumber: string
+    acesssionNumber: string // NOTE this is misspelled
     docketNumbers: string[]
 
     description: string

src/util/download.ts

@@ -0,0 +1,75 @@
+//  https://elibrary.ferc.gov/eLibrarywebapi/api/File/DownloadP8File
+
+import { writeFile } from 'fs'
+import { join } from 'path'
+import { DownloadFileParams } from '~/types/DownloadFile'
+
+export type FileInfo = {
+  path: string
+  file: string
+  format: string
+}
+
+const FileInfoDefault: FileInfo = {
+  path: 'temp',
+  format: 'pdf',
+  file: 'temp',
+}
+
+export async function downloadFile(
+  params: DownloadFileParams,
+  fileInfo?: Partial<FileInfo>
+) {
+  const fi = {
+    ...FileInfoDefault,
+    ...{ file: params.fileidLst[0] ?? FileInfoDefault.file },
+    ...fileInfo,
+  } as FileInfo
+
+  return download(fi, 'File/DownloadP8File', {}, params)
+}
+
+export async function generatePDF(
+  accessionNumber: string,
+  fileInfo?: Partial<FileInfo>
+) {
+  const fi = {
+    ...FileInfoDefault,
+    ...{ file: accessionNumber },
+    ...fileInfo,
+  } as FileInfo
+
+  return download(
+    fi,
+    'File/DownloadPDF',
+    { accessionNumber },
+    '{serverLocation: ""}'
+  )
+}
+
+async function download<
+  B extends Record<string, unknown> | string,
+  P extends Record<string, string>
+>(fileInfo: FileInfo, url: string, params: P, body: B) {
+  fetch(
+    `https://elibrary.ferc.gov/eLibrarywebapi/api/${url}?` +
+      new URLSearchParams(params),
+    {
+      headers: {
+        accept: 'application/json, text/plain, */*',
+        'content-type': 'application/json',
+      },
+      body: typeof body === 'string' ? body : JSON.stringify(body),
+      method: 'POST',
+    }
+  )
+    .then((r) => r.arrayBuffer())
+    .then((ab) =>
+      writeFile(
+        join(fileInfo.path, `${fileInfo.file}.${fileInfo.format}`),
+        Buffer.from(ab),
+        'binary',
+        () => {}
+      )
+    )
+}

src/util/fetch.ts

@@ -0,0 +1,13 @@
+export function fetchFunction<SearchParams, SearchResults>(
+  searchType: string
+): (params: SearchParams) => Promise<SearchResults> {
+  return async (params: SearchParams) => {
+    return fetch(`https://elibrary.ferc.gov/eLibrarywebapi/api/${searchType}`, {
+      headers: {
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify(params),
+      method: 'post',
+    }).then((r) => r.json())
+  }
+}

test/consts/generalSearch.ts

@@ -1,3 +1,5 @@
+import { GeneralSearchParams } from '~/types'
+
 export const EMPTY_GENERAL_SEARCH = {
   searchHits: [],
   totalHits: 0,
@@ -83,3 +85,34 @@ export const NULL_SEARCH_RESULT = {
   errorMessage: null,
   searchResultId: null,
 }
+
+export const SINGLE_PAGE_SEARCH_PARAMS: GeneralSearchParams = {
+  searchText: '*',
+  searchFullText: true,
+  searchDescription: true,
+  dateSearches: [
+    {
+      dateType: 'filed_date',
+      startDate: '2020-11-19',
+      endDate: '2020-12-16',
+    },
+  ],
+  availability: null,
+  affiliations: [],
+  categories: [],
+  libraries: [],
+  accessionNumber: null,
+  eFiling: false,
+  docketSearches: [
+    {
+      docketNumber: 'P-15056-000',
+      subDocketNumbers: [],
+    },
+  ],
+  resultsPerPage: 1,
+  classTypes: [],
+  sortBy: '',
+  groupBy: 'NONE',
+  idolResultID: '',
+  allDates: false,
+}

test/download.test.ts

@@ -0,0 +1,36 @@
+import { existsSync } from 'fs'
+import { describe, expect, test } from 'vitest'
+import { downloadFile } from '~/util/download'
+
+describe.concurrent('Download Files', () => {
+  test('Download PDF', async () => {
+    const fileID = '020AAB97-66E2-5005-8110-C31FAFC91712'
+    await downloadFile({
+      FileType: '',
+      accession: '',
+      fileid: 0,
+      FileIDAll: '',
+      fileidLst: [fileID],
+      Islegacy: false,
+    })
+    expect(existsSync(`temp/${fileID}`)).toBeTruthy()
+  })
+
+  test('Download TIF', async () => {
+    const fileID = '020D2E72-66E2-5005-8110-C31FAFC91712'
+    await downloadFile(
+      {
+        FileType: '',
+        accession: '',
+        fileid: 0,
+        FileIDAll: '',
+        fileidLst: [fileID],
+        Islegacy: false,
+      },
+      {
+        format: 'tif',
+      }
+    )
+    expect(existsSync(`temp/${fileID}`)).toBeTruthy()
+  })
+})

test/index.test.ts

@@ -1,10 +1,10 @@
-import { bench, describe, expect, test } from 'vitest'
+import { describe, expect, test } from 'vitest'
 import GeneralSearch from '../src/search/general'
-import { GeneralSearchParams, GeneralSearchResult } from '../src/types'
 import {
   EMPTY_GENERAL_SEARCH,
   NULL_SEARCH_RESULT,
   SINGLE_GENERAL_SEARCH,
+  SINGLE_PAGE_SEARCH_PARAMS,
 } from './consts/generalSearch'
 
 describe.concurrent('General Search', () => {
@@ -17,7 +17,7 @@ describe.concurrent('General Search', () => {
         },
       ],
     })
-    await s.fetch()
+    await s.getData()
     expect(s.data).toEqual(EMPTY_GENERAL_SEARCH)
   })
 
@@ -37,7 +37,7 @@ describe.concurrent('General Search', () => {
         },
       ],
     })
-    await s.fetch()
+    await s.getData()
     expect(s.data).toEqual(SINGLE_GENERAL_SEARCH)
   })
   test('Null Docket Number', async () => {
@@ -49,43 +49,15 @@ describe.concurrent('General Search', () => {
         },
       ],
     })
-    await s.fetch()
+    await s.getData()
     expect(s.data).toEqual(NULL_SEARCH_RESULT)
   })
-  test('Null Docket Number', async () => {
-    const s = new GeneralSearch({
-      docketSearches: [
-        {
-          docketNumber: '',
-          subDocketNumbers: [],
-        },
-      ],
-      resultsPerPage: 200,
-    })
-    await s.fetch()
-
-    const keys: (keyof GeneralSearchResult['searchHits'][number])[] = [
-      'acesssionNumber',
-      'familyValue',
-      'docketNumbers',
-      'score',
-    ]
-    const sets = Object.fromEntries(keys.map((k) => [k, new Set()]))
-
-    s.data?.searchHits.forEach((e) => {
-      keys.forEach((k) =>
-        Array.isArray(e[k]) && typeof e[k] !== 'string'
-          ? e[k]?.forEach((ek) => sets[k].add(ek))
-          : sets[k].add(e[k])
-      )
-    })
-    console.log(
-      JSON.stringify(
-        Object.fromEntries(
-          Object.entries(sets).map(([key, val]) => [key, [...val]])
-        )
-      )
-    )
-    // expect(s.data).toEqual(NULL_SEARCH_RESULT)
+  test('Single Page Search', async () => {
+    const s = new GeneralSearch(SINGLE_PAGE_SEARCH_PARAMS)
+    await s.getData()
+    expect(await s.prevPage()).toBe(false)
+    expect(await s.nextPage()).toBe(true)
+    expect(await s.nextPage()).toBe(false)
+    expect(await s.prevPage()).toBe(true)
   })
 })