@axetroy/crawler
Options
All
  • Public
  • Public/Protected
  • All
Menu

Index

Enumerations

Classes

Interfaces

Type aliases

Variables

Functions

Type aliases

Body

Body: string | JSONValue | Buffer | ArrayBuffer | ArrayBufferView | Uint8Array | URLSearchParams | Stream | null

HTTPHeaders

HTTPHeaders: IncomingHttpHeaders

JSONObject

JSONObject: {}

Type declaration

JSONPrimitive

JSONPrimitive: string | number | boolean | null

JSONValue

JSONValue: JSONPrimitive | JSONObject | JSONArray

SubscribeFn

SubscribeFn: (task: Task) => Promise<void>

Type declaration

    • (task: Task): Promise<void>

    • Parameters

      Returns Promise<void>

TaskType

TaskType: "request" | "download"

Url

Url: string | UrlCustomer

Variables

combine

combine: combine

Const dirPath

dirPath: string = path.join(process.cwd(), ".crawler")

Const errorLog

errorLog: string = path.join(logDirPath, "error.log")

Let id

id: number = 0

Const infoLog

infoLog: string = path.join(logDirPath, "info.log")

Const logDirPath

logDirPath: string = path.join(dirPath, "logs")

Const logger

logger: Logger = winston.createLogger({levels: winston.config.syslog.levels,format: combine(timestamp(), createFormatter()),transports: [new winston.transports.File({ filename: infoLog, level: "info" }),new winston.transports.File({ filename: errorLog, level: "error" })]})

Const numCPUs

numCPUs: number = os.cpus().length

printf

printf: printf

timestamp

timestamp: timestamp

Functions

createFormatter

  • createFormatter(color?: boolean): Format

createResponse

  • createResponse(response: AxiosResponse): Response

default

  • default(provider: Provider, options: Options): void

request

sleep

  • sleep(ms: number): Promise<void>

sum

  • sum(array: number[]): any