openapi.yml

openapi: 3.0.2
info:
  title: Arachnio API
  description: Arachnio is an API product for creating structured data from the web.
  contact:
    email: api@arachn.io
  version: 0.1.5
servers:
  - url: https://api.arachn.io/v1
tags:
  - name: link
    description: Endpoints for processing hyperlinks
  - name: domain
    description: Endpoints for processing domain names
paths:
  /domains/parse:
    post:
      tags:
        - domain
      summary: Parse a domain name
      description: Parses a domain name into its component parts
      operationId: parseDomainName
      requestBody:
        description: The domain name to parse
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DomainName'
      responses:
        200:
          description: The domain name was parsed successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ParsedDomainName'
  /domains/parse/batch:
    post:
      tags:
        - domain
      summary: Parse a batch of domain names
      description: Parses a batch of domain names into their component parts
      operationId: parseDomainNameBatch
      requestBody:
        description: The batch of domain names to parse
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DomainNameBatch'
      responses:
        200:
          description: The domain names were parsed successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ParsedDomainNameBatch'
  /links/parse:
    post:
      tags:
        - link
      summary: Parse a link
      description: Parses an absolute hyperlink into its component parts
      operationId: parseLink
      requestBody:
        description: The link to parse
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/Link'
      responses:
        200:
          description: The link was parsed successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ParsedLink'
  /links/parse/batch:
    post:
      tags:
        - link
      summary: Parse a batch of links
      description: Parses a batch of absolute hyperlinks into their component parts
      operationId: parseLinkBatch
      requestBody:
        description: The batch of links to parse
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/LinkBatch'
      responses:
        200:
          description: The links were parsed successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ParsedLinkBatch'
  /links/unwind:
    post:
      tags:
        - link
      summary: Unwind a link
      description: Converts a link to its maximally unwound, and possibly canonical, form. For example, convert `https://nyti.ms/3rBgKkH` to `https://www.nytimes.com/2019/07/17/science/apollo-11-landing-anniversary.html`.
      operationId: unwindLink
      x-async-enabled: true
      requestBody:
        description: The link to unwind
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/Link'
        required: true
      responses:
        200:
          description: The link was canonicalized successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UnwoundLink'
  /links/unwind/batch:
    post:
      tags:
        - link
      summary: Unwind a batch of links
      description: Converts a batch of links to their maximally unwound, and possibly canonical, forms
      operationId: unwindLinkBatch
      x-async-enabled: true
      requestBody:
        description: The batch of links to unwind
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/LinkBatch'
        required: true
      responses:
        200:
          description: The link was canonicalized successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UnwoundLinkBatch'
  /links/extract:
    post:
      tags:
        - link
      summary: Extract a link
      description: Unwinds a link and attempts to extract structured microdata and body content
      operationId: extractLink
      x-async-enabled: true
      requestBody:
        description: The link to extract
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/Link'
        required: true
      responses:
        200:
          description: The link was extracted successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ExtractedLink'
components:
  schemas:
    Scheme:
      description: An application-level protocol for exchanging hypermedia documents, namely http and https.
      type: string
      enum:
        - http
        - https
        
    Authority:
      description: A URL authority consisting of a required host and optional port.
      type: object
      required:
        - host
      properties:
        host:
          $ref: '#/components/schemas/Host'
        port:
          description: A TCP port number
          example: 80
          type: integer
          minimum: 0
          maximum: 65353
          format: int32
          
    Host:
      description: The coordinate of a specific destination on the internet
      type: object
      required:
        - type
      properties:
        type:
          description: "The type of this host, one of: `domain`, `ipv4`, `ipv6`"
          example: domain
          type: string
      discriminator:
        propertyName: type
        mapping:
          domain: '#/components/schemas/DomainNameHost'
          ipv4: '#/components/schemas/IpV4AddressHost'
          ipv6: '#/components/schemas/IpV6AddressHost'
          
    DomainNameHost:
      allOf:
        - $ref: '#/components/schemas/Host'
        - description: A domain name coordinate typically resolved with DNS, e.g. www.google.com
          type: object
          properties:
            domain:
              $ref: '#/components/schemas/ParsedDomainName'
          
    IpV4AddressHost:
      allOf:
        - $ref: '#/components/schemas/Host'
        - description: An IPv4 address
          example: 8.8.8.8
          type: object
          required:
            - address
          properties:
            address:
              type: string
          
    IpV6AddressHost:
      allOf:
        - $ref: '#/components/schemas/Host'
        - description: An IPv6 address, e.g. 
          example: 2001:4860:4860::8888
          type: object
          required:
            - address
          properties:
            address:
              type: string
          
    QueryParameter:
      description: A URL query parameter, e.g. `foo=bar`. All fields are urldecoded.
      type: object
      required:
        - name
      properties:
        name:
          description: The name of the query parameter
          example: foo
          type: string
        value:
          description: The value of the query parameter
          example: bar
          type: string

    ParsedDomainName:
      type: object
      required:
        - registrySuffix
        - publicSuffix
        - hostname
      properties:
        registrySuffix:
          description: The top-level domain of this hostname
          example: com
          type: string
        publicSuffix:
          description: The top-level public suffix of this hostname
          example: google.com
          type: string
        hostname:
          description: The complete domain name
          example: www.google.com
          type: string
          format: hostname
          
    ParsedLink:
      description: A URL that has been parsed into its constituent parts
      type: object
      required:
        - link
        - scheme
        - authority
        - path
      properties:
        link:
          description: The original link
          example: https://www.nytimes.com/2019/07/17/science/apollo-11-landing-anniversary.html
          type: string
          format: url
        scheme:
          $ref: '#/components/schemas/Scheme'
        authority:
          $ref: '#/components/schemas/Authority'
        path:
          description: The path from the original link
          example: /2019/07/17/science/apollo-11-landing-anniversary.html
          type: string
        queryParameters:
          description: The list of query parameters from the original link
          type: array
          items:
            $ref: '#/components/schemas/QueryParameter'

    UnwindingOutcome:
      description: |
        A description of the final outcome of unwinding a link
        
        * `success2xx` -- The final hop completed with a 2xx status code. This typically indicates a successful unwinding.
        * `failure4xx` -- The unwinding failed due to a 4xx status code.
        * `failure5xx` -- The unwinding failed due to a 5xx status code.
        * `timeout` -- The unwinding failed because it took too long.
        * `tooManyHops` -- The unwinding failed due to too many hops.
        * `protocolError` -- The unwinding failed because the server(s) did not follow the HTTP protocol.
        * `networkingError` -- The unwinding failed due to an I/O error.
      type: string
      enum:
        - success2xx
        - failure4xx
        - failure5xx
        - timeout
        - tooManyHops
        - protocolError
        - networkingError

    UnwoundLink:
      description: The result of a link unwinding operation
      type: object
      required:
        - original
        - unwound
        - canonical
      properties:
        original:
          $ref: '#/components/schemas/ParsedLink'
        unwound:
          $ref: '#/components/schemas/ParsedLink'
        outcome:
          $ref: '#/components/schemas/UnwindingOutcome'
        canonical:
          description: Either `true` if the unwound URL is canonical, or `false` otherwise. A URL is canonical if and only if the outcome is `success2xx` and the final entity is an HTML document that contains a canonical meta link.
          example: true
          type: boolean
          
    ImageMetadata:
      description: An extracted image
      type: object
      properties:
        url:
          description: The location of the file
          example: https://www.example.com/path/to/image.jpg
          type: string
          format: url
        width:
          description: The natural width of the image, in pixels
          example: 640
          type: integer
          format: int32
        height:
          description: The natural height of the image, in pixels
          example: 480
          type: integer
          format: int32
          
    EntityMetadata:
      description: A description of an extracted page
      type: object
      properties:
        entityType:
          description: The type of this page, e.g., "webpage". For reasons of future-proofing, applications should ignore values they do not recognize.
          example: "webpage"
          type: string
      discriminator:
        propertyName: entityType
        mapping:
          other: '#/components/schemas/OtherEntityMetadata'
          webpage: '#/components/schemas/WebpageEntityMetadata'
          
    OtherEntityMetadata:
      allOf:
        - $ref: '#/components/schemas/EntityMetadata'
        - type: object

    Locale:
      type: object
      properties:
        language:
          type: string
          description: An ISO 639-1 language code
          example: "en"
          maxLength: 2
        country:
          type: string
          description: An ISO 3166-1 Alpha 2 country code
          example: "US"
          maxLength: 2
          
    MetaTag:
      type: object
      additionalProperties:
        type: string
          
    WebpageEntityMetadata:
      allOf:
        - $ref: '#/components/schemas/EntityMetadata'
        - type: object
          properties:
            webpageType:
              description: The type of this webpage, e.g., "article". For reasons of future-proofing, applications should ignore values they do not recognize.
              example: "article"
              type: string
            locale:
              $ref: '#/components/schemas/Locale'
            title:
              description: The title of the page
              example: "Men Walk On Moon"
              type: string
              maxLength: 256
            metaTags:
              type: array
              items:
                $ref: '#/components/schemas/MetaTag'
            thumbnail:
              $ref: '#/components/schemas/ImageMetadata'
            description:
              description: A description of the content of the page.
              example: "Houston, Monday, July 21--Men have landed and walked on the moon. Two Americans, astronauts of Apollo 11, steered their fragile four-legged lunar module safely and smoothly to the historic landing yesterday at 4:17:40 P.M., Eastern daylight time."
              type: string
              maxLength: 1024
      discriminator:
        propertyName: webpageType
        mapping:
          article: '#/components/schemas/ArticleWebpageEntityMetadata'
          other: '#/components/schemas/OtherWebpageEntityMetadata'
          
    OtherWebpageEntityMetadata:
      allOf:
        - $ref: '#/components/schemas/WebpageEntityMetadata'
        - type: object
        
    Hyperlink:
      type: object
      properties:
        href:
          $ref: '#/components/schemas/ParsedLink'
        rel:
          type: string
          description: The value of the `rel` attribute, e.g., `nofollow`, `author`, etc.
        outlink:
          type: boolean
          description: Whether or not this link's destination points to a separate site
        anchorText:
          type: string
          description: The text content of this link's tag
          
    ArticleAuthor:
      type: object
      properties:
        name:
          description: The author's name
          example: "John Noble Wilford"
          type: string

    ArticleWebpageEntityMetadata:
      allOf:
        - $ref: '#/components/schemas/WebpageEntityMetadata'
        - type: object
          properties:
            keywords:
              type: array
              items:
                type: string
            author:
              $ref: '#/components/schemas/ArticleAuthor'
            publishedAt:
              description: The timestamp when the page was originally published.
              example: "1969-07-21T00:00:00Z"
              type: string
              format: date-time
            modifiedAt:
              description: The timestamp when the page was most recently modified.
              example: "1969-07-21T00:00:00Z"
              type: string
              format: date-time
            bodyHtml:
              description: The sanitized contents of the page body. It may be truncated. Will always be a valid HTML fragment.
              type: string
              maxLength: 32768
            bodyText:
              description: The contents of the page body. It may be truncated.
              type: string
              maxLength: 32768
            bodyLinks:
              description: The list of hyperlinks embedded in this page's body content.
              type: array
              items:
                $ref: '#/components/schemas/Hyperlink'
            
    ExtractedLink:
      description: A description of an extracted link
      type: object
      required:
        - link
        - metadata
      properties:
        link:
          $ref: '#/components/schemas/UnwoundLink'
        entity:
          $ref: '#/components/schemas/EntityMetadata'

    DomainName:
      description: A domain name for parsing, etc.
      type: object
      required:
        - hostname
      properties:
        hostname:
          description: The hostname to parse
          example: www.google.com
          type: string
          format: hostname
          nullable: false
          maxLength: 253

    DomainNameBatchEntry:
      description: An entry in a batch of domain names
      type: object
      required:
        - id
        - domain
      properties:
        id:
          description: An ID for this entry. It must be unique within its batch.
          type: string
          maxLength: 80
        domain:
          $ref: '#/components/schemas/DomainName'

    DomainNameBatch:
      description: A batch of domain names
      type: object
      required:
        - entries
      properties:
        entries:
          description: The domain names
          type: array
          minLength: 1
          maxLength: 100
          items:
            $ref: '#/components/schemas/DomainNameBatchEntry'
            
    ParsedDomainNameBatchEntry:
      description: An entry in a batch of parsed domain names
      type: object
      required:
        - id
        - parsedDomain
      properties:
        id:
          description: An ID for this entry. It must be unique within its batch.
          type: string
          maxLength: 80
        parsedDomain:
          $ref: '#/components/schemas/ParsedDomainName'

    ParsedDomainNameBatch:
      description: A batch of parsed domain names
      type: object
      required:
        - entries
      properties:
        entries:
          description: The parsed domain names
          type: array
          minLength: 1
          maxLength: 10
          items:
            $ref: '#/components/schemas/ParsedDomainNameBatchEntry'
            
    Link:
      description: A hyperlink for parsing, extracting, etc.
      type: object
      required:
        - url
      properties:
        url:
          description: The hyperlink
          example: https://www.nytimes.com/2019/07/17/science/apollo-11-landing-anniversary.html
          type: string
          format: url
          nullable: false
          maxLength: 2048
          
    LinkBatchEntry:
      description: An entry in a batch of links
      type: object
      required:
        - id
        - link
      properties:
        id:
          description: An ID for this entry. It must be unique within its batch.
          type: string
          maxLength: 80
        link:
          $ref: '#/components/schemas/Link'

    LinkBatch:
      description: A batch of links
      type: object
      required:
        - entries
      properties:
        entries:
          description: The links
          type: array
          minLength: 1
          maxLength: 100
          items:
            $ref: '#/components/schemas/LinkBatchEntry'

    ParsedLinkBatchEntry:
      description: An entry in a batch of parsed links
      type: object
      required:
        - id
        - parsedLink
      properties:
        id:
          description: An ID for this entry. It must be unique within its batch.
          type: string
          maxLength: 80
        parsedLink:
          $ref: '#/components/schemas/ParsedLink'

    ParsedLinkBatch:
      description: A batch of parsed links
      type: object
      required:
        - entries
      properties:
        entries:
          description: The parsed links
          type: array
          minLength: 1
          maxLength: 10
          items:
            $ref: '#/components/schemas/ParsedLinkBatchEntry'
            
    UnwoundLinkBatchEntry:
      description: An entry in a batch of parsed links
      type: object
      required:
        - id
        - unwoundLink
      properties:
        id:
          type: string
          maxLength: 80
        unwoundLink:
          $ref: '#/components/schemas/UnwoundLink'

    UnwoundLinkBatch:
      description: A batch of unwound links
      type: object
      required:
        - entries
      properties:
        entries:
          description: The unwound links
          type: array
          minLength: 1
          maxLength: 10
          items:
            $ref: '#/components/schemas/UnwoundLinkBatchEntry'