reddit-extract-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zstandard\n",
    "import os \n",
    "import json\n",
    "import sys\n",
    "import csv\n",
    "from datetime import datetime\n",
    "import logging.handlers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processed 100,000 lines: 0 bad lines : 0.36% complete\n",
      "Processed 100,000 lines: 0 bad lines : 0.36% complete\n",
      "Processed 100,000 lines: 0 bad lines : 0.36% complete\n",
      "Processed 200,000 lines: 0 bad lines : 0.71% complete\n",
      "Processed 200,000 lines: 0 bad lines : 0.71% complete\n",
      "Processed 200,000 lines: 0 bad lines : 0.71% complete\n",
      "Processed 300,000 lines: 0 bad lines : 1.04% complete\n",
      "Processed 300,000 lines: 0 bad lines : 1.04% complete\n",
      "Processed 300,000 lines: 0 bad lines : 1.04% complete\n",
      "Processed 400,000 lines: 0 bad lines : 1.26% complete\n",
      "Processed 400,000 lines: 0 bad lines : 1.26% complete\n",
      "Processed 400,000 lines: 0 bad lines : 1.26% complete\n",
      "Processed 500,000 lines: 0 bad lines : 1.57% complete\n",
      "Processed 500,000 lines: 0 bad lines : 1.57% complete\n",
      "Processed 500,000 lines: 0 bad lines : 1.57% complete\n",
      "Processed 600,000 lines: 0 bad lines : 1.91% complete\n",
      "Processed 600,000 lines: 0 bad lines : 1.91% complete\n",
      "Processed 600,000 lines: 0 bad lines : 1.91% complete\n",
      "Processed 700,000 lines: 0 bad lines : 2.26% complete\n",
      "Processed 700,000 lines: 0 bad lines : 2.26% complete\n",
      "Processed 700,000 lines: 0 bad lines : 2.26% complete\n",
      "Processed 800,000 lines: 0 bad lines : 2.59% complete\n",
      "Processed 800,000 lines: 0 bad lines : 2.59% complete\n",
      "Processed 800,000 lines: 0 bad lines : 2.59% complete\n",
      "Processed 900,000 lines: 0 bad lines : 2.92% complete\n",
      "Processed 900,000 lines: 0 bad lines : 2.92% complete\n",
      "Processed 900,000 lines: 0 bad lines : 2.92% complete\n",
      "Processed 1,000,000 lines: 0 bad lines : 3.14% complete\n",
      "Processed 1,000,000 lines: 0 bad lines : 3.14% complete\n",
      "Processed 1,000,000 lines: 0 bad lines : 3.14% complete\n",
      "Processed 1,100,000 lines: 0 bad lines : 3.47% complete\n",
      "Processed 1,100,000 lines: 0 bad lines : 3.47% complete\n",
      "Processed 1,100,000 lines: 0 bad lines : 3.47% complete\n",
      "Processed 1,200,000 lines: 0 bad lines : 3.78% complete\n",
      "Processed 1,200,000 lines: 0 bad lines : 3.78% complete\n",
      "Processed 1,200,000 lines: 0 bad lines : 3.78% complete\n",
      "Processed 1,300,000 lines: 0 bad lines : 4.09% complete\n",
      "Processed 1,300,000 lines: 0 bad lines : 4.09% complete\n",
      "Processed 1,300,000 lines: 0 bad lines : 4.09% complete\n",
      "Processed 1,400,000 lines: 0 bad lines : 4.39% complete\n",
      "Processed 1,400,000 lines: 0 bad lines : 4.39% complete\n",
      "Processed 1,400,000 lines: 0 bad lines : 4.39% complete\n",
      "Processed 1,500,000 lines: 0 bad lines : 4.70% complete\n",
      "Processed 1,500,000 lines: 0 bad lines : 4.70% complete\n",
      "Processed 1,500,000 lines: 0 bad lines : 4.70% complete\n",
      "Processed 1,600,000 lines: 0 bad lines : 5.02% complete\n",
      "Processed 1,600,000 lines: 0 bad lines : 5.02% complete\n",
      "Processed 1,600,000 lines: 0 bad lines : 5.02% complete\n",
      "Processed 1,700,000 lines: 0 bad lines : 5.34% complete\n",
      "Processed 1,700,000 lines: 0 bad lines : 5.34% complete\n",
      "Processed 1,700,000 lines: 0 bad lines : 5.34% complete\n",
      "Processed 1,800,000 lines: 0 bad lines : 5.67% complete\n",
      "Processed 1,800,000 lines: 0 bad lines : 5.67% complete\n",
      "Processed 1,800,000 lines: 0 bad lines : 5.67% complete\n",
      "Processed 1,900,000 lines: 0 bad lines : 5.99% complete\n",
      "Processed 1,900,000 lines: 0 bad lines : 5.99% complete\n",
      "Processed 1,900,000 lines: 0 bad lines : 5.99% complete\n",
      "Processed 2,000,000 lines: 0 bad lines : 6.32% complete\n",
      "Processed 2,000,000 lines: 0 bad lines : 6.32% complete\n",
      "Processed 2,000,000 lines: 0 bad lines : 6.32% complete\n",
      "Processed 2,100,000 lines: 0 bad lines : 6.53% complete\n",
      "Processed 2,100,000 lines: 0 bad lines : 6.53% complete\n",
      "Processed 2,100,000 lines: 0 bad lines : 6.53% complete\n",
      "Processed 2,200,000 lines: 0 bad lines : 6.84% complete\n",
      "Processed 2,200,000 lines: 0 bad lines : 6.84% complete\n",
      "Processed 2,200,000 lines: 0 bad lines : 6.84% complete\n",
      "Processed 2,300,000 lines: 0 bad lines : 7.15% complete\n",
      "Processed 2,300,000 lines: 0 bad lines : 7.15% complete\n",
      "Processed 2,300,000 lines: 0 bad lines : 7.15% complete\n",
      "Processed 2,400,000 lines: 0 bad lines : 7.47% complete\n",
      "Processed 2,400,000 lines: 0 bad lines : 7.47% complete\n",
      "Processed 2,400,000 lines: 0 bad lines : 7.47% complete\n",
      "Processed 2,500,000 lines: 0 bad lines : 7.80% complete\n",
      "Processed 2,500,000 lines: 0 bad lines : 7.80% complete\n",
      "Processed 2,500,000 lines: 0 bad lines : 7.80% complete\n",
      "Processed 2,600,000 lines: 0 bad lines : 8.13% complete\n",
      "Processed 2,600,000 lines: 0 bad lines : 8.13% complete\n",
      "Processed 2,600,000 lines: 0 bad lines : 8.13% complete\n",
      "Processed 2,700,000 lines: 0 bad lines : 8.47% complete\n",
      "Processed 2,700,000 lines: 0 bad lines : 8.47% complete\n",
      "Processed 2,700,000 lines: 0 bad lines : 8.47% complete\n",
      "Processed 2,800,000 lines: 0 bad lines : 8.69% complete\n",
      "Processed 2,800,000 lines: 0 bad lines : 8.69% complete\n",
      "Processed 2,800,000 lines: 0 bad lines : 8.69% complete\n",
      "Processed 2,900,000 lines: 0 bad lines : 9.03% complete\n",
      "Processed 2,900,000 lines: 0 bad lines : 9.03% complete\n",
      "Processed 2,900,000 lines: 0 bad lines : 9.03% complete\n",
      "Processed 3,000,000 lines: 0 bad lines : 9.36% complete\n",
      "Processed 3,000,000 lines: 0 bad lines : 9.36% complete\n",
      "Processed 3,000,000 lines: 0 bad lines : 9.36% complete\n",
      "Processed 3,100,000 lines: 0 bad lines : 9.69% complete\n",
      "Processed 3,100,000 lines: 0 bad lines : 9.69% complete\n",
      "Processed 3,100,000 lines: 0 bad lines : 9.69% complete\n",
      "Processed 3,200,000 lines: 0 bad lines : 9.90% complete\n",
      "Processed 3,200,000 lines: 0 bad lines : 9.90% complete\n",
      "Processed 3,200,000 lines: 0 bad lines : 9.90% complete\n",
      "Processed 3,300,000 lines: 0 bad lines : 10.22% complete\n",
      "Processed 3,300,000 lines: 0 bad lines : 10.22% complete\n",
      "Processed 3,300,000 lines: 0 bad lines : 10.22% complete\n",
      "Processed 3,400,000 lines: 0 bad lines : 10.54% complete\n",
      "Processed 3,400,000 lines: 0 bad lines : 10.54% complete\n",
      "Processed 3,400,000 lines: 0 bad lines : 10.54% complete\n",
      "Processed 3,500,000 lines: 0 bad lines : 10.88% complete\n",
      "Processed 3,500,000 lines: 0 bad lines : 10.88% complete\n",
      "Processed 3,500,000 lines: 0 bad lines : 10.88% complete\n",
      "Processed 3,600,000 lines: 0 bad lines : 11.21% complete\n",
      "Processed 3,600,000 lines: 0 bad lines : 11.21% complete\n",
      "Processed 3,600,000 lines: 0 bad lines : 11.21% complete\n",
      "Processed 3,700,000 lines: 0 bad lines : 11.54% complete\n",
      "Processed 3,700,000 lines: 0 bad lines : 11.54% complete\n",
      "Processed 3,700,000 lines: 0 bad lines : 11.54% complete\n",
      "Processed 3,800,000 lines: 0 bad lines : 11.76% complete\n",
      "Processed 3,800,000 lines: 0 bad lines : 11.76% complete\n",
      "Processed 3,800,000 lines: 0 bad lines : 11.76% complete\n",
      "Processed 3,900,000 lines: 0 bad lines : 12.09% complete\n",
      "Processed 3,900,000 lines: 0 bad lines : 12.09% complete\n",
      "Processed 3,900,000 lines: 0 bad lines : 12.09% complete\n",
      "Processed 4,000,000 lines: 0 bad lines : 12.43% complete\n",
      "Processed 4,000,000 lines: 0 bad lines : 12.43% complete\n",
      "Processed 4,000,000 lines: 0 bad lines : 12.43% complete\n",
      "Processed 4,100,000 lines: 0 bad lines : 12.75% complete\n",
      "Processed 4,100,000 lines: 0 bad lines : 12.75% complete\n",
      "Processed 4,100,000 lines: 0 bad lines : 12.75% complete\n",
      "Processed 4,200,000 lines: 0 bad lines : 13.07% complete\n",
      "Processed 4,200,000 lines: 0 bad lines : 13.07% complete\n",
      "Processed 4,200,000 lines: 0 bad lines : 13.07% complete\n",
      "Processed 4,300,000 lines: 0 bad lines : 13.27% complete\n",
      "Processed 4,300,000 lines: 0 bad lines : 13.27% complete\n",
      "Processed 4,300,000 lines: 0 bad lines : 13.27% complete\n",
      "Processed 4,400,000 lines: 0 bad lines : 13.57% complete\n",
      "Processed 4,400,000 lines: 0 bad lines : 13.57% complete\n",
      "Processed 4,400,000 lines: 0 bad lines : 13.57% complete\n",
      "Processed 4,500,000 lines: 0 bad lines : 13.89% complete\n",
      "Processed 4,500,000 lines: 0 bad lines : 13.89% complete\n",
      "Processed 4,500,000 lines: 0 bad lines : 13.89% complete\n",
      "Processed 4,600,000 lines: 0 bad lines : 14.21% complete\n",
      "Processed 4,600,000 lines: 0 bad lines : 14.21% complete\n",
      "Processed 4,600,000 lines: 0 bad lines : 14.21% complete\n",
      "Processed 4,700,000 lines: 0 bad lines : 14.54% complete\n",
      "Processed 4,700,000 lines: 0 bad lines : 14.54% complete\n",
      "Processed 4,700,000 lines: 0 bad lines : 14.54% complete\n",
      "Processed 4,800,000 lines: 0 bad lines : 14.86% complete\n",
      "Processed 4,800,000 lines: 0 bad lines : 14.86% complete\n",
      "Processed 4,800,000 lines: 0 bad lines : 14.86% complete\n",
      "Processed 4,900,000 lines: 0 bad lines : 15.19% complete\n",
      "Processed 4,900,000 lines: 0 bad lines : 15.19% complete\n",
      "Processed 4,900,000 lines: 0 bad lines : 15.19% complete\n",
      "Processed 5,000,000 lines: 0 bad lines : 15.51% complete\n",
      "Processed 5,000,000 lines: 0 bad lines : 15.51% complete\n",
      "Processed 5,000,000 lines: 0 bad lines : 15.51% complete\n",
      "Processed 5,100,000 lines: 0 bad lines : 15.84% complete\n",
      "Processed 5,100,000 lines: 0 bad lines : 15.84% complete\n",
      "Processed 5,100,000 lines: 0 bad lines : 15.84% complete\n",
      "Processed 5,200,000 lines: 0 bad lines : 16.06% complete\n",
      "Processed 5,200,000 lines: 0 bad lines : 16.06% complete\n",
      "Processed 5,200,000 lines: 0 bad lines : 16.06% complete\n",
      "Processed 5,300,000 lines: 0 bad lines : 16.38% complete\n",
      "Processed 5,300,000 lines: 0 bad lines : 16.38% complete\n",
      "Processed 5,300,000 lines: 0 bad lines : 16.38% complete\n",
      "Processed 5,400,000 lines: 0 bad lines : 16.70% complete\n",
      "Processed 5,400,000 lines: 0 bad lines : 16.70% complete\n",
      "Processed 5,400,000 lines: 0 bad lines : 16.70% complete\n",
      "Processed 5,500,000 lines: 0 bad lines : 17.01% complete\n",
      "Processed 5,500,000 lines: 0 bad lines : 17.01% complete\n",
      "Processed 5,500,000 lines: 0 bad lines : 17.01% complete\n",
      "Processed 5,600,000 lines: 0 bad lines : 17.34% complete\n",
      "Processed 5,600,000 lines: 0 bad lines : 17.34% complete\n",
      "Processed 5,600,000 lines: 0 bad lines : 17.34% complete\n",
      "Processed 5,700,000 lines: 0 bad lines : 17.66% complete\n",
      "Processed 5,700,000 lines: 0 bad lines : 17.66% complete\n",
      "Processed 5,700,000 lines: 0 bad lines : 17.66% complete\n",
      "Processed 5,800,000 lines: 0 bad lines : 17.99% complete\n",
      "Processed 5,800,000 lines: 0 bad lines : 17.99% complete\n",
      "Processed 5,800,000 lines: 0 bad lines : 17.99% complete\n",
      "Processed 5,900,000 lines: 0 bad lines : 18.21% complete\n",
      "Processed 5,900,000 lines: 0 bad lines : 18.21% complete\n",
      "Processed 5,900,000 lines: 0 bad lines : 18.21% complete\n",
      "Processed 6,000,000 lines: 0 bad lines : 18.53% complete\n",
      "Processed 6,000,000 lines: 0 bad lines : 18.53% complete\n",
      "Processed 6,000,000 lines: 0 bad lines : 18.53% complete\n",
      "Processed 6,100,000 lines: 0 bad lines : 18.85% complete\n",
      "Processed 6,100,000 lines: 0 bad lines : 18.85% complete\n",
      "Processed 6,100,000 lines: 0 bad lines : 18.85% complete\n",
      "Processed 6,200,000 lines: 0 bad lines : 19.17% complete\n",
      "Processed 6,200,000 lines: 0 bad lines : 19.17% complete\n",
      "Processed 6,200,000 lines: 0 bad lines : 19.17% complete\n",
      "Processed 6,300,000 lines: 0 bad lines : 19.49% complete\n",
      "Processed 6,300,000 lines: 0 bad lines : 19.49% complete\n",
      "Processed 6,300,000 lines: 0 bad lines : 19.49% complete\n",
      "Processed 6,400,000 lines: 0 bad lines : 19.71% complete\n",
      "Processed 6,400,000 lines: 0 bad lines : 19.71% complete\n",
      "Processed 6,400,000 lines: 0 bad lines : 19.71% complete\n",
      "Processed 6,500,000 lines: 0 bad lines : 20.03% complete\n",
      "Processed 6,500,000 lines: 0 bad lines : 20.03% complete\n",
      "Processed 6,500,000 lines: 0 bad lines : 20.03% complete\n",
      "Processed 6,600,000 lines: 0 bad lines : 20.35% complete\n",
      "Processed 6,600,000 lines: 0 bad lines : 20.35% complete\n",
      "Processed 6,600,000 lines: 0 bad lines : 20.35% complete\n",
      "Processed 6,700,000 lines: 0 bad lines : 20.68% complete\n",
      "Processed 6,700,000 lines: 0 bad lines : 20.68% complete\n",
      "Processed 6,700,000 lines: 0 bad lines : 20.68% complete\n",
      "Processed 6,800,000 lines: 0 bad lines : 21.01% complete\n",
      "Processed 6,800,000 lines: 0 bad lines : 21.01% complete\n",
      "Processed 6,800,000 lines: 0 bad lines : 21.01% complete\n",
      "Processed 6,900,000 lines: 0 bad lines : 21.33% complete\n",
      "Processed 6,900,000 lines: 0 bad lines : 21.33% complete\n",
      "Processed 6,900,000 lines: 0 bad lines : 21.33% complete\n",
      "Processed 7,000,000 lines: 0 bad lines : 21.54% complete\n",
      "Processed 7,000,000 lines: 0 bad lines : 21.54% complete\n",
      "Processed 7,000,000 lines: 0 bad lines : 21.54% complete\n",
      "Processed 7,100,000 lines: 0 bad lines : 21.87% complete\n",
      "Processed 7,100,000 lines: 0 bad lines : 21.87% complete\n",
      "Processed 7,100,000 lines: 0 bad lines : 21.87% complete\n",
      "Processed 7,200,000 lines: 0 bad lines : 22.19% complete\n",
      "Processed 7,200,000 lines: 0 bad lines : 22.19% complete\n",
      "Processed 7,200,000 lines: 0 bad lines : 22.19% complete\n",
      "Processed 7,300,000 lines: 0 bad lines : 22.52% complete\n",
      "Processed 7,300,000 lines: 0 bad lines : 22.52% complete\n",
      "Processed 7,300,000 lines: 0 bad lines : 22.52% complete\n",
      "Processed 7,400,000 lines: 0 bad lines : 22.85% complete\n",
      "Processed 7,400,000 lines: 0 bad lines : 22.85% complete\n",
      "Processed 7,400,000 lines: 0 bad lines : 22.85% complete\n",
      "Processed 7,500,000 lines: 0 bad lines : 23.17% complete\n",
      "Processed 7,500,000 lines: 0 bad lines : 23.17% complete\n",
      "Processed 7,500,000 lines: 0 bad lines : 23.17% complete\n",
      "Processed 7,600,000 lines: 0 bad lines : 23.38% complete\n",
      "Processed 7,600,000 lines: 0 bad lines : 23.38% complete\n",
      "Processed 7,600,000 lines: 0 bad lines : 23.38% complete\n",
      "Processed 7,700,000 lines: 0 bad lines : 23.70% complete\n",
      "Processed 7,700,000 lines: 0 bad lines : 23.70% complete\n",
      "Processed 7,700,000 lines: 0 bad lines : 23.70% complete\n",
      "Processed 7,800,000 lines: 0 bad lines : 24.03% complete\n",
      "Processed 7,800,000 lines: 0 bad lines : 24.03% complete\n",
      "Processed 7,800,000 lines: 0 bad lines : 24.03% complete\n",
      "Processed 7,900,000 lines: 0 bad lines : 24.36% complete\n",
      "Processed 7,900,000 lines: 0 bad lines : 24.36% complete\n",
      "Processed 7,900,000 lines: 0 bad lines : 24.36% complete\n",
      "Processed 8,000,000 lines: 0 bad lines : 24.69% complete\n",
      "Processed 8,000,000 lines: 0 bad lines : 24.69% complete\n",
      "Processed 8,000,000 lines: 0 bad lines : 24.69% complete\n",
      "Processed 8,100,000 lines: 0 bad lines : 25.02% complete\n",
      "Processed 8,100,000 lines: 0 bad lines : 25.02% complete\n",
      "Processed 8,100,000 lines: 0 bad lines : 25.02% complete\n",
      "Processed 8,200,000 lines: 0 bad lines : 25.34% complete\n",
      "Processed 8,200,000 lines: 0 bad lines : 25.34% complete\n",
      "Processed 8,200,000 lines: 0 bad lines : 25.34% complete\n",
      "Processed 8,300,000 lines: 0 bad lines : 25.67% complete\n",
      "Processed 8,300,000 lines: 0 bad lines : 25.67% complete\n",
      "Processed 8,300,000 lines: 0 bad lines : 25.67% complete\n",
      "Processed 8,400,000 lines: 0 bad lines : 25.89% complete\n",
      "Processed 8,400,000 lines: 0 bad lines : 25.89% complete\n",
      "Processed 8,400,000 lines: 0 bad lines : 25.89% complete\n",
      "Processed 8,500,000 lines: 0 bad lines : 26.20% complete\n",
      "Processed 8,500,000 lines: 0 bad lines : 26.20% complete\n",
      "Processed 8,500,000 lines: 0 bad lines : 26.20% complete\n",
      "Processed 8,600,000 lines: 0 bad lines : 26.51% complete\n",
      "Processed 8,600,000 lines: 0 bad lines : 26.51% complete\n",
      "Processed 8,600,000 lines: 0 bad lines : 26.51% complete\n",
      "Processed 8,700,000 lines: 0 bad lines : 26.83% complete\n",
      "Processed 8,700,000 lines: 0 bad lines : 26.83% complete\n",
      "Processed 8,700,000 lines: 0 bad lines : 26.83% complete\n",
      "Processed 8,800,000 lines: 0 bad lines : 27.16% complete\n",
      "Processed 8,800,000 lines: 0 bad lines : 27.16% complete\n",
      "Processed 8,800,000 lines: 0 bad lines : 27.16% complete\n",
      "Processed 8,900,000 lines: 0 bad lines : 27.50% complete\n",
      "Processed 8,900,000 lines: 0 bad lines : 27.50% complete\n",
      "Processed 8,900,000 lines: 0 bad lines : 27.50% complete\n",
      "Processed 9,000,000 lines: 0 bad lines : 27.83% complete\n",
      "Processed 9,000,000 lines: 0 bad lines : 27.83% complete\n",
      "Processed 9,000,000 lines: 0 bad lines : 27.83% complete\n",
      "Processed 9,100,000 lines: 0 bad lines : 28.17% complete\n",
      "Processed 9,100,000 lines: 0 bad lines : 28.17% complete\n",
      "Processed 9,100,000 lines: 0 bad lines : 28.17% complete\n",
      "Processed 9,200,000 lines: 0 bad lines : 28.39% complete\n",
      "Processed 9,200,000 lines: 0 bad lines : 28.39% complete\n",
      "Processed 9,200,000 lines: 0 bad lines : 28.39% complete\n",
      "Processed 9,300,000 lines: 0 bad lines : 28.73% complete\n",
      "Processed 9,300,000 lines: 0 bad lines : 28.73% complete\n",
      "Processed 9,300,000 lines: 0 bad lines : 28.73% complete\n",
      "Processed 9,400,000 lines: 0 bad lines : 29.05% complete\n",
      "Processed 9,400,000 lines: 0 bad lines : 29.05% complete\n",
      "Processed 9,400,000 lines: 0 bad lines : 29.05% complete\n",
      "Processed 9,500,000 lines: 0 bad lines : 29.37% complete\n",
      "Processed 9,500,000 lines: 0 bad lines : 29.37% complete\n",
      "Processed 9,500,000 lines: 0 bad lines : 29.37% complete\n",
      "Processed 9,600,000 lines: 0 bad lines : 29.69% complete\n",
      "Processed 9,600,000 lines: 0 bad lines : 29.69% complete\n",
      "Processed 9,600,000 lines: 0 bad lines : 29.69% complete\n",
      "Processed 9,700,000 lines: 0 bad lines : 30.02% complete\n",
      "Processed 9,700,000 lines: 0 bad lines : 30.02% complete\n",
      "Processed 9,700,000 lines: 0 bad lines : 30.02% complete\n",
      "Processed 9,800,000 lines: 0 bad lines : 30.25% complete\n",
      "Processed 9,800,000 lines: 0 bad lines : 30.25% complete\n",
      "Processed 9,800,000 lines: 0 bad lines : 30.25% complete\n",
      "Processed 9,900,000 lines: 0 bad lines : 30.58% complete\n",
      "Processed 9,900,000 lines: 0 bad lines : 30.58% complete\n",
      "Processed 9,900,000 lines: 0 bad lines : 30.58% complete\n",
      "Processed 10,000,000 lines: 0 bad lines : 30.92% complete\n",
      "Processed 10,000,000 lines: 0 bad lines : 30.92% complete\n",
      "Processed 10,000,000 lines: 0 bad lines : 30.92% complete\n",
      "Processed 10,100,000 lines: 0 bad lines : 31.26% complete\n",
      "Processed 10,100,000 lines: 0 bad lines : 31.26% complete\n",
      "Processed 10,100,000 lines: 0 bad lines : 31.26% complete\n",
      "Processed 10,200,000 lines: 0 bad lines : 31.59% complete\n",
      "Processed 10,200,000 lines: 0 bad lines : 31.59% complete\n",
      "Processed 10,200,000 lines: 0 bad lines : 31.59% complete\n",
      "Processed 10,300,000 lines: 0 bad lines : 31.92% complete\n",
      "Processed 10,300,000 lines: 0 bad lines : 31.92% complete\n",
      "Processed 10,300,000 lines: 0 bad lines : 31.92% complete\n",
      "Processed 10,400,000 lines: 0 bad lines : 32.14% complete\n",
      "Processed 10,400,000 lines: 0 bad lines : 32.14% complete\n",
      "Processed 10,400,000 lines: 0 bad lines : 32.14% complete\n",
      "Processed 10,500,000 lines: 0 bad lines : 32.46% complete\n",
      "Processed 10,500,000 lines: 0 bad lines : 32.46% complete\n",
      "Processed 10,500,000 lines: 0 bad lines : 32.46% complete\n",
      "Processed 10,600,000 lines: 0 bad lines : 32.78% complete\n",
      "Processed 10,600,000 lines: 0 bad lines : 32.78% complete\n",
      "Processed 10,600,000 lines: 0 bad lines : 32.78% complete\n",
      "Processed 10,700,000 lines: 0 bad lines : 33.12% complete\n",
      "Processed 10,700,000 lines: 0 bad lines : 33.12% complete\n",
      "Processed 10,700,000 lines: 0 bad lines : 33.12% complete\n",
      "Processed 10,800,000 lines: 0 bad lines : 33.45% complete\n",
      "Processed 10,800,000 lines: 0 bad lines : 33.45% complete\n",
      "Processed 10,800,000 lines: 0 bad lines : 33.45% complete\n",
      "Processed 10,900,000 lines: 0 bad lines : 33.79% complete\n",
      "Processed 10,900,000 lines: 0 bad lines : 33.79% complete\n",
      "Processed 10,900,000 lines: 0 bad lines : 33.79% complete\n",
      "Processed 11,000,000 lines: 0 bad lines : 34.12% complete\n",
      "Processed 11,000,000 lines: 0 bad lines : 34.12% complete\n",
      "Processed 11,000,000 lines: 0 bad lines : 34.12% complete\n",
      "Processed 11,100,000 lines: 0 bad lines : 34.46% complete\n",
      "Processed 11,100,000 lines: 0 bad lines : 34.46% complete\n",
      "Processed 11,100,000 lines: 0 bad lines : 34.46% complete\n",
      "Processed 11,200,000 lines: 0 bad lines : 34.68% complete\n",
      "Processed 11,200,000 lines: 0 bad lines : 34.68% complete\n",
      "Processed 11,200,000 lines: 0 bad lines : 34.68% complete\n",
      "Processed 11,300,000 lines: 0 bad lines : 35.01% complete\n",
      "Processed 11,300,000 lines: 0 bad lines : 35.01% complete\n",
      "Processed 11,300,000 lines: 0 bad lines : 35.01% complete\n",
      "Processed 11,400,000 lines: 0 bad lines : 35.34% complete\n",
      "Processed 11,400,000 lines: 0 bad lines : 35.34% complete\n",
      "Processed 11,400,000 lines: 0 bad lines : 35.34% complete\n",
      "Processed 11,500,000 lines: 0 bad lines : 35.66% complete\n",
      "Processed 11,500,000 lines: 0 bad lines : 35.66% complete\n",
      "Processed 11,500,000 lines: 0 bad lines : 35.66% complete\n",
      "Processed 11,600,000 lines: 0 bad lines : 35.98% complete\n",
      "Processed 11,600,000 lines: 0 bad lines : 35.98% complete\n",
      "Processed 11,600,000 lines: 0 bad lines : 35.98% complete\n",
      "Processed 11,700,000 lines: 0 bad lines : 36.31% complete\n",
      "Processed 11,700,000 lines: 0 bad lines : 36.31% complete\n",
      "Processed 11,700,000 lines: 0 bad lines : 36.31% complete\n",
      "Processed 11,800,000 lines: 0 bad lines : 36.54% complete\n",
      "Processed 11,800,000 lines: 0 bad lines : 36.54% complete\n",
      "Processed 11,800,000 lines: 0 bad lines : 36.54% complete\n",
      "Processed 11,900,000 lines: 0 bad lines : 36.87% complete\n",
      "Processed 11,900,000 lines: 0 bad lines : 36.87% complete\n",
      "Processed 11,900,000 lines: 0 bad lines : 36.87% complete\n",
      "Processed 12,000,000 lines: 0 bad lines : 37.21% complete\n",
      "Processed 12,000,000 lines: 0 bad lines : 37.21% complete\n",
      "Processed 12,000,000 lines: 0 bad lines : 37.21% complete\n",
      "Processed 12,100,000 lines: 0 bad lines : 37.54% complete\n",
      "Processed 12,100,000 lines: 0 bad lines : 37.54% complete\n",
      "Processed 12,100,000 lines: 0 bad lines : 37.54% complete\n",
      "Processed 12,200,000 lines: 0 bad lines : 37.88% complete\n",
      "Processed 12,200,000 lines: 0 bad lines : 37.88% complete\n",
      "Processed 12,200,000 lines: 0 bad lines : 37.88% complete\n",
      "Processed 12,300,000 lines: 0 bad lines : 38.21% complete\n",
      "Processed 12,300,000 lines: 0 bad lines : 38.21% complete\n",
      "Processed 12,300,000 lines: 0 bad lines : 38.21% complete\n",
      "Processed 12,400,000 lines: 0 bad lines : 38.43% complete\n",
      "Processed 12,400,000 lines: 0 bad lines : 38.43% complete\n",
      "Processed 12,400,000 lines: 0 bad lines : 38.43% complete\n",
      "Processed 12,500,000 lines: 0 bad lines : 38.76% complete\n",
      "Processed 12,500,000 lines: 0 bad lines : 38.76% complete\n",
      "Processed 12,500,000 lines: 0 bad lines : 38.76% complete\n",
      "Processed 12,600,000 lines: 0 bad lines : 39.08% complete\n",
      "Processed 12,600,000 lines: 0 bad lines : 39.08% complete\n",
      "Processed 12,600,000 lines: 0 bad lines : 39.08% complete\n",
      "Processed 12,700,000 lines: 0 bad lines : 39.39% complete\n",
      "Processed 12,700,000 lines: 0 bad lines : 39.39% complete\n",
      "Processed 12,700,000 lines: 0 bad lines : 39.39% complete\n",
      "Processed 12,800,000 lines: 0 bad lines : 39.71% complete\n",
      "Processed 12,800,000 lines: 0 bad lines : 39.71% complete\n",
      "Processed 12,800,000 lines: 0 bad lines : 39.71% complete\n",
      "Processed 12,900,000 lines: 0 bad lines : 40.04% complete\n",
      "Processed 12,900,000 lines: 0 bad lines : 40.04% complete\n",
      "Processed 12,900,000 lines: 0 bad lines : 40.04% complete\n",
      "Processed 13,000,000 lines: 0 bad lines : 40.26% complete\n",
      "Processed 13,000,000 lines: 0 bad lines : 40.26% complete\n",
      "Processed 13,000,000 lines: 0 bad lines : 40.26% complete\n",
      "Processed 13,100,000 lines: 0 bad lines : 40.60% complete\n",
      "Processed 13,100,000 lines: 0 bad lines : 40.60% complete\n",
      "Processed 13,100,000 lines: 0 bad lines : 40.60% complete\n",
      "Processed 13,200,000 lines: 0 bad lines : 40.93% complete\n",
      "Processed 13,200,000 lines: 0 bad lines : 40.93% complete\n",
      "Processed 13,200,000 lines: 0 bad lines : 40.93% complete\n",
      "Processed 13,300,000 lines: 0 bad lines : 41.26% complete\n",
      "Processed 13,300,000 lines: 0 bad lines : 41.26% complete\n",
      "Processed 13,300,000 lines: 0 bad lines : 41.26% complete\n",
      "Processed 13,400,000 lines: 0 bad lines : 41.60% complete\n",
      "Processed 13,400,000 lines: 0 bad lines : 41.60% complete\n",
      "Processed 13,400,000 lines: 0 bad lines : 41.60% complete\n",
      "Processed 13,500,000 lines: 0 bad lines : 41.81% complete\n",
      "Processed 13,500,000 lines: 0 bad lines : 41.81% complete\n",
      "Processed 13,500,000 lines: 0 bad lines : 41.81% complete\n",
      "Processed 13,600,000 lines: 0 bad lines : 42.14% complete\n",
      "Processed 13,600,000 lines: 0 bad lines : 42.14% complete\n",
      "Processed 13,600,000 lines: 0 bad lines : 42.14% complete\n",
      "Processed 13,700,000 lines: 0 bad lines : 42.46% complete\n",
      "Processed 13,700,000 lines: 0 bad lines : 42.46% complete\n",
      "Processed 13,700,000 lines: 0 bad lines : 42.46% complete\n",
      "Processed 13,800,000 lines: 0 bad lines : 42.78% complete\n",
      "Processed 13,800,000 lines: 0 bad lines : 42.78% complete\n",
      "Processed 13,800,000 lines: 0 bad lines : 42.78% complete\n",
      "Processed 13,900,000 lines: 0 bad lines : 43.11% complete\n",
      "Processed 13,900,000 lines: 0 bad lines : 43.11% complete\n",
      "Processed 13,900,000 lines: 0 bad lines : 43.11% complete\n",
      "Processed 14,000,000 lines: 0 bad lines : 43.45% complete\n",
      "Processed 14,000,000 lines: 0 bad lines : 43.45% complete\n",
      "Processed 14,000,000 lines: 0 bad lines : 43.45% complete\n",
      "Processed 14,100,000 lines: 0 bad lines : 43.79% complete\n",
      "Processed 14,100,000 lines: 0 bad lines : 43.79% complete\n",
      "Processed 14,100,000 lines: 0 bad lines : 43.79% complete\n",
      "Processed 14,200,000 lines: 0 bad lines : 44.11% complete\n",
      "Processed 14,200,000 lines: 0 bad lines : 44.11% complete\n",
      "Processed 14,200,000 lines: 0 bad lines : 44.11% complete\n",
      "Processed 14,300,000 lines: 0 bad lines : 44.34% complete\n",
      "Processed 14,300,000 lines: 0 bad lines : 44.34% complete\n",
      "Processed 14,300,000 lines: 0 bad lines : 44.34% complete\n",
      "Processed 14,400,000 lines: 0 bad lines : 44.67% complete\n",
      "Processed 14,400,000 lines: 0 bad lines : 44.67% complete\n",
      "Processed 14,400,000 lines: 0 bad lines : 44.67% complete\n",
      "Processed 14,500,000 lines: 0 bad lines : 45.00% complete\n",
      "Processed 14,500,000 lines: 0 bad lines : 45.00% complete\n",
      "Processed 14,500,000 lines: 0 bad lines : 45.00% complete\n",
      "Processed 14,600,000 lines: 0 bad lines : 45.33% complete\n",
      "Processed 14,600,000 lines: 0 bad lines : 45.33% complete\n",
      "Processed 14,600,000 lines: 0 bad lines : 45.33% complete\n",
      "Processed 14,700,000 lines: 0 bad lines : 45.65% complete\n",
      "Processed 14,700,000 lines: 0 bad lines : 45.65% complete\n",
      "Processed 14,700,000 lines: 0 bad lines : 45.65% complete\n",
      "Processed 14,800,000 lines: 0 bad lines : 45.97% complete\n",
      "Processed 14,800,000 lines: 0 bad lines : 45.97% complete\n",
      "Processed 14,800,000 lines: 0 bad lines : 45.97% complete\n",
      "Processed 14,900,000 lines: 0 bad lines : 46.18% complete\n",
      "Processed 14,900,000 lines: 0 bad lines : 46.18% complete\n",
      "Processed 14,900,000 lines: 0 bad lines : 46.18% complete\n",
      "Processed 15,000,000 lines: 0 bad lines : 46.52% complete\n",
      "Processed 15,000,000 lines: 0 bad lines : 46.52% complete\n",
      "Processed 15,000,000 lines: 0 bad lines : 46.52% complete\n",
      "Processed 15,100,000 lines: 0 bad lines : 46.86% complete\n",
      "Processed 15,100,000 lines: 0 bad lines : 46.86% complete\n",
      "Processed 15,100,000 lines: 0 bad lines : 46.86% complete\n",
      "Processed 15,200,000 lines: 0 bad lines : 47.19% complete\n",
      "Processed 15,200,000 lines: 0 bad lines : 47.19% complete\n",
      "Processed 15,200,000 lines: 0 bad lines : 47.19% complete\n",
      "Processed 15,300,000 lines: 0 bad lines : 47.53% complete\n",
      "Processed 15,300,000 lines: 0 bad lines : 47.53% complete\n",
      "Processed 15,300,000 lines: 0 bad lines : 47.53% complete\n",
      "Processed 15,400,000 lines: 0 bad lines : 47.74% complete\n",
      "Processed 15,400,000 lines: 0 bad lines : 47.74% complete\n",
      "Processed 15,400,000 lines: 0 bad lines : 47.74% complete\n",
      "Processed 15,500,000 lines: 0 bad lines : 48.07% complete\n",
      "Processed 15,500,000 lines: 0 bad lines : 48.07% complete\n",
      "Processed 15,500,000 lines: 0 bad lines : 48.07% complete\n",
      "Processed 15,600,000 lines: 0 bad lines : 48.40% complete\n",
      "Processed 15,600,000 lines: 0 bad lines : 48.40% complete\n",
      "Processed 15,600,000 lines: 0 bad lines : 48.40% complete\n",
      "Processed 15,700,000 lines: 0 bad lines : 48.72% complete\n",
      "Processed 15,700,000 lines: 0 bad lines : 48.72% complete\n",
      "Processed 15,700,000 lines: 0 bad lines : 48.72% complete\n",
      "Processed 15,800,000 lines: 0 bad lines : 49.03% complete\n",
      "Processed 15,800,000 lines: 0 bad lines : 49.03% complete\n",
      "Processed 15,800,000 lines: 0 bad lines : 49.03% complete\n",
      "Processed 15,900,000 lines: 0 bad lines : 49.37% complete\n",
      "Processed 15,900,000 lines: 0 bad lines : 49.37% complete\n",
      "Processed 15,900,000 lines: 0 bad lines : 49.37% complete\n",
      "Processed 16,000,000 lines: 0 bad lines : 49.59% complete\n",
      "Processed 16,000,000 lines: 0 bad lines : 49.59% complete\n",
      "Processed 16,000,000 lines: 0 bad lines : 49.59% complete\n",
      "Processed 16,100,000 lines: 0 bad lines : 49.92% complete\n",
      "Processed 16,100,000 lines: 0 bad lines : 49.92% complete\n",
      "Processed 16,100,000 lines: 0 bad lines : 49.92% complete\n",
      "Processed 16,200,000 lines: 0 bad lines : 50.26% complete\n",
      "Processed 16,200,000 lines: 0 bad lines : 50.26% complete\n",
      "Processed 16,200,000 lines: 0 bad lines : 50.26% complete\n",
      "Processed 16,300,000 lines: 0 bad lines : 50.59% complete\n",
      "Processed 16,300,000 lines: 0 bad lines : 50.59% complete\n",
      "Processed 16,300,000 lines: 0 bad lines : 50.59% complete\n",
      "Processed 16,400,000 lines: 0 bad lines : 50.93% complete\n",
      "Processed 16,400,000 lines: 0 bad lines : 50.93% complete\n",
      "Processed 16,400,000 lines: 0 bad lines : 50.93% complete\n",
      "Processed 16,500,000 lines: 0 bad lines : 51.15% complete\n",
      "Processed 16,500,000 lines: 0 bad lines : 51.15% complete\n",
      "Processed 16,500,000 lines: 0 bad lines : 51.15% complete\n",
      "Processed 16,600,000 lines: 0 bad lines : 51.47% complete\n",
      "Processed 16,600,000 lines: 0 bad lines : 51.47% complete\n",
      "Processed 16,600,000 lines: 0 bad lines : 51.47% complete\n",
      "Processed 16,700,000 lines: 0 bad lines : 51.78% complete\n",
      "Processed 16,700,000 lines: 0 bad lines : 51.78% complete\n",
      "Processed 16,700,000 lines: 0 bad lines : 51.78% complete\n",
      "Processed 16,800,000 lines: 0 bad lines : 52.10% complete\n",
      "Processed 16,800,000 lines: 0 bad lines : 52.10% complete\n",
      "Processed 16,800,000 lines: 0 bad lines : 52.10% complete\n",
      "Processed 16,900,000 lines: 0 bad lines : 52.43% complete\n",
      "Processed 16,900,000 lines: 0 bad lines : 52.43% complete\n",
      "Processed 16,900,000 lines: 0 bad lines : 52.43% complete\n",
      "Processed 17,000,000 lines: 0 bad lines : 52.76% complete\n",
      "Processed 17,000,000 lines: 0 bad lines : 52.76% complete\n",
      "Processed 17,000,000 lines: 0 bad lines : 52.76% complete\n",
      "Processed 17,100,000 lines: 0 bad lines : 53.10% complete\n",
      "Processed 17,100,000 lines: 0 bad lines : 53.10% complete\n",
      "Processed 17,100,000 lines: 0 bad lines : 53.10% complete\n",
      "Processed 17,200,000 lines: 0 bad lines : 53.43% complete\n",
      "Processed 17,200,000 lines: 0 bad lines : 53.43% complete\n",
      "Processed 17,200,000 lines: 0 bad lines : 53.43% complete\n",
      "Processed 17,300,000 lines: 0 bad lines : 53.65% complete\n",
      "Processed 17,300,000 lines: 0 bad lines : 53.65% complete\n",
      "Processed 17,300,000 lines: 0 bad lines : 53.65% complete\n",
      "Processed 17,400,000 lines: 0 bad lines : 53.98% complete\n",
      "Processed 17,400,000 lines: 0 bad lines : 53.98% complete\n",
      "Processed 17,400,000 lines: 0 bad lines : 53.98% complete\n",
      "Processed 17,500,000 lines: 0 bad lines : 54.31% complete\n",
      "Processed 17,500,000 lines: 0 bad lines : 54.31% complete\n",
      "Processed 17,500,000 lines: 0 bad lines : 54.31% complete\n",
      "Processed 17,600,000 lines: 0 bad lines : 54.64% complete\n",
      "Processed 17,600,000 lines: 0 bad lines : 54.64% complete\n",
      "Processed 17,600,000 lines: 0 bad lines : 54.64% complete\n",
      "Processed 17,700,000 lines: 0 bad lines : 54.96% complete\n",
      "Processed 17,700,000 lines: 0 bad lines : 54.96% complete\n",
      "Processed 17,700,000 lines: 0 bad lines : 54.96% complete\n",
      "Processed 17,800,000 lines: 0 bad lines : 55.28% complete\n",
      "Processed 17,800,000 lines: 0 bad lines : 55.28% complete\n",
      "Processed 17,800,000 lines: 0 bad lines : 55.28% complete\n",
      "Processed 17,900,000 lines: 0 bad lines : 55.50% complete\n",
      "Processed 17,900,000 lines: 0 bad lines : 55.50% complete\n",
      "Processed 17,900,000 lines: 0 bad lines : 55.50% complete\n",
      "Processed 18,000,000 lines: 0 bad lines : 55.83% complete\n",
      "Processed 18,000,000 lines: 0 bad lines : 55.83% complete\n",
      "Processed 18,000,000 lines: 0 bad lines : 55.83% complete\n",
      "Processed 18,100,000 lines: 0 bad lines : 56.16% complete\n",
      "Processed 18,100,000 lines: 0 bad lines : 56.16% complete\n",
      "Processed 18,100,000 lines: 0 bad lines : 56.16% complete\n",
      "Processed 18,200,000 lines: 0 bad lines : 56.51% complete\n",
      "Processed 18,200,000 lines: 0 bad lines : 56.51% complete\n",
      "Processed 18,200,000 lines: 0 bad lines : 56.51% complete\n",
      "Processed 18,300,000 lines: 0 bad lines : 56.84% complete\n",
      "Processed 18,300,000 lines: 0 bad lines : 56.84% complete\n",
      "Processed 18,300,000 lines: 0 bad lines : 56.84% complete\n",
      "Processed 18,400,000 lines: 0 bad lines : 57.18% complete\n",
      "Processed 18,400,000 lines: 0 bad lines : 57.18% complete\n",
      "Processed 18,400,000 lines: 0 bad lines : 57.18% complete\n",
      "Processed 18,500,000 lines: 0 bad lines : 57.51% complete\n",
      "Processed 18,500,000 lines: 0 bad lines : 57.51% complete\n",
      "Processed 18,500,000 lines: 0 bad lines : 57.51% complete\n",
      "Processed 18,600,000 lines: 0 bad lines : 57.72% complete\n",
      "Processed 18,600,000 lines: 0 bad lines : 57.72% complete\n",
      "Processed 18,600,000 lines: 0 bad lines : 57.72% complete\n",
      "Processed 18,700,000 lines: 0 bad lines : 58.05% complete\n",
      "Processed 18,700,000 lines: 0 bad lines : 58.05% complete\n",
      "Processed 18,700,000 lines: 0 bad lines : 58.05% complete\n",
      "Processed 18,800,000 lines: 0 bad lines : 58.37% complete\n",
      "Processed 18,800,000 lines: 0 bad lines : 58.37% complete\n",
      "Processed 18,800,000 lines: 0 bad lines : 58.37% complete\n",
      "Processed 18,900,000 lines: 0 bad lines : 58.70% complete\n",
      "Processed 18,900,000 lines: 0 bad lines : 58.70% complete\n",
      "Processed 18,900,000 lines: 0 bad lines : 58.70% complete\n",
      "Processed 19,000,000 lines: 0 bad lines : 59.03% complete\n",
      "Processed 19,000,000 lines: 0 bad lines : 59.03% complete\n",
      "Processed 19,000,000 lines: 0 bad lines : 59.03% complete\n",
      "Processed 19,100,000 lines: 0 bad lines : 59.37% complete\n",
      "Processed 19,100,000 lines: 0 bad lines : 59.37% complete\n",
      "Processed 19,100,000 lines: 0 bad lines : 59.37% complete\n",
      "Processed 19,200,000 lines: 0 bad lines : 59.71% complete\n",
      "Processed 19,200,000 lines: 0 bad lines : 59.71% complete\n",
      "Processed 19,200,000 lines: 0 bad lines : 59.71% complete\n",
      "Processed 19,300,000 lines: 0 bad lines : 59.94% complete\n",
      "Processed 19,300,000 lines: 0 bad lines : 59.94% complete\n",
      "Processed 19,300,000 lines: 0 bad lines : 59.94% complete\n",
      "Processed 19,400,000 lines: 0 bad lines : 60.27% complete\n",
      "Processed 19,400,000 lines: 0 bad lines : 60.27% complete\n",
      "Processed 19,400,000 lines: 0 bad lines : 60.27% complete\n",
      "Processed 19,500,000 lines: 0 bad lines : 60.61% complete\n",
      "Processed 19,500,000 lines: 0 bad lines : 60.61% complete\n",
      "Processed 19,500,000 lines: 0 bad lines : 60.61% complete\n",
      "Processed 19,600,000 lines: 0 bad lines : 60.94% complete\n",
      "Processed 19,600,000 lines: 0 bad lines : 60.94% complete\n",
      "Processed 19,600,000 lines: 0 bad lines : 60.94% complete\n",
      "Processed 19,700,000 lines: 0 bad lines : 61.15% complete\n",
      "Processed 19,700,000 lines: 0 bad lines : 61.15% complete\n",
      "Processed 19,700,000 lines: 0 bad lines : 61.15% complete\n",
      "Processed 19,800,000 lines: 0 bad lines : 61.48% complete\n",
      "Processed 19,800,000 lines: 0 bad lines : 61.48% complete\n",
      "Processed 19,800,000 lines: 0 bad lines : 61.48% complete\n",
      "Processed 19,900,000 lines: 0 bad lines : 61.80% complete\n",
      "Processed 19,900,000 lines: 0 bad lines : 61.80% complete\n",
      "Processed 19,900,000 lines: 0 bad lines : 61.80% complete\n",
      "Processed 20,000,000 lines: 0 bad lines : 62.13% complete\n",
      "Processed 20,000,000 lines: 0 bad lines : 62.13% complete\n",
      "Processed 20,000,000 lines: 0 bad lines : 62.13% complete\n",
      "Processed 20,100,000 lines: 0 bad lines : 62.46% complete\n",
      "Processed 20,100,000 lines: 0 bad lines : 62.46% complete\n",
      "Processed 20,100,000 lines: 0 bad lines : 62.46% complete\n",
      "Processed 20,200,000 lines: 0 bad lines : 62.79% complete\n",
      "Processed 20,200,000 lines: 0 bad lines : 62.79% complete\n",
      "Processed 20,200,000 lines: 0 bad lines : 62.79% complete\n",
      "Processed 20,300,000 lines: 0 bad lines : 63.02% complete\n",
      "Processed 20,300,000 lines: 0 bad lines : 63.02% complete\n",
      "Processed 20,300,000 lines: 0 bad lines : 63.02% complete\n",
      "Processed 20,400,000 lines: 0 bad lines : 63.35% complete\n",
      "Processed 20,400,000 lines: 0 bad lines : 63.35% complete\n",
      "Processed 20,400,000 lines: 0 bad lines : 63.35% complete\n",
      "Processed 20,500,000 lines: 0 bad lines : 63.69% complete\n",
      "Processed 20,500,000 lines: 0 bad lines : 63.69% complete\n",
      "Processed 20,500,000 lines: 0 bad lines : 63.69% complete\n",
      "Processed 20,600,000 lines: 0 bad lines : 64.02% complete\n",
      "Processed 20,600,000 lines: 0 bad lines : 64.02% complete\n",
      "Processed 20,600,000 lines: 0 bad lines : 64.02% complete\n",
      "Processed 20,700,000 lines: 0 bad lines : 64.35% complete\n",
      "Processed 20,700,000 lines: 0 bad lines : 64.35% complete\n",
      "Processed 20,700,000 lines: 0 bad lines : 64.35% complete\n",
      "Processed 20,800,000 lines: 0 bad lines : 64.57% complete\n",
      "Processed 20,800,000 lines: 0 bad lines : 64.57% complete\n",
      "Processed 20,800,000 lines: 0 bad lines : 64.57% complete\n",
      "Processed 20,900,000 lines: 0 bad lines : 64.90% complete\n",
      "Processed 20,900,000 lines: 0 bad lines : 64.90% complete\n",
      "Processed 20,900,000 lines: 0 bad lines : 64.90% complete\n",
      "Processed 21,000,000 lines: 0 bad lines : 65.22% complete\n",
      "Processed 21,000,000 lines: 0 bad lines : 65.22% complete\n",
      "Processed 21,000,000 lines: 0 bad lines : 65.22% complete\n",
      "Processed 21,100,000 lines: 0 bad lines : 65.56% complete\n",
      "Processed 21,100,000 lines: 0 bad lines : 65.56% complete\n",
      "Processed 21,100,000 lines: 0 bad lines : 65.56% complete\n",
      "Processed 21,200,000 lines: 0 bad lines : 65.90% complete\n",
      "Processed 21,200,000 lines: 0 bad lines : 65.90% complete\n",
      "Processed 21,200,000 lines: 0 bad lines : 65.90% complete\n",
      "Processed 21,300,000 lines: 0 bad lines : 66.24% complete\n",
      "Processed 21,300,000 lines: 0 bad lines : 66.24% complete\n",
      "Processed 21,300,000 lines: 0 bad lines : 66.24% complete\n",
      "Processed 21,400,000 lines: 0 bad lines : 66.46% complete\n",
      "Processed 21,400,000 lines: 0 bad lines : 66.46% complete\n",
      "Processed 21,400,000 lines: 0 bad lines : 66.46% complete\n",
      "Processed 21,500,000 lines: 0 bad lines : 66.80% complete\n",
      "Processed 21,500,000 lines: 0 bad lines : 66.80% complete\n",
      "Processed 21,500,000 lines: 0 bad lines : 66.80% complete\n",
      "Processed 21,600,000 lines: 0 bad lines : 67.13% complete\n",
      "Processed 21,600,000 lines: 0 bad lines : 67.13% complete\n",
      "Processed 21,600,000 lines: 0 bad lines : 67.13% complete\n",
      "Processed 21,700,000 lines: 0 bad lines : 67.46% complete\n",
      "Processed 21,700,000 lines: 0 bad lines : 67.46% complete\n",
      "Processed 21,700,000 lines: 0 bad lines : 67.46% complete\n",
      "Processed 21,800,000 lines: 0 bad lines : 67.79% complete\n",
      "Processed 21,800,000 lines: 0 bad lines : 67.79% complete\n",
      "Processed 21,800,000 lines: 0 bad lines : 67.79% complete\n",
      "Processed 21,900,000 lines: 0 bad lines : 68.01% complete\n",
      "Processed 21,900,000 lines: 0 bad lines : 68.01% complete\n",
      "Processed 21,900,000 lines: 0 bad lines : 68.01% complete\n",
      "Processed 22,000,000 lines: 0 bad lines : 68.33% complete\n",
      "Processed 22,000,000 lines: 0 bad lines : 68.33% complete\n",
      "Processed 22,000,000 lines: 0 bad lines : 68.33% complete\n",
      "Processed 22,100,000 lines: 0 bad lines : 68.65% complete\n",
      "Processed 22,100,000 lines: 0 bad lines : 68.65% complete\n",
      "Processed 22,100,000 lines: 0 bad lines : 68.65% complete\n",
      "Processed 22,200,000 lines: 0 bad lines : 68.98% complete\n",
      "Processed 22,200,000 lines: 0 bad lines : 68.98% complete\n",
      "Processed 22,200,000 lines: 0 bad lines : 68.98% complete\n",
      "Processed 22,300,000 lines: 0 bad lines : 69.32% complete\n",
      "Processed 22,300,000 lines: 0 bad lines : 69.32% complete\n",
      "Processed 22,300,000 lines: 0 bad lines : 69.32% complete\n",
      "Processed 22,400,000 lines: 0 bad lines : 69.66% complete\n",
      "Processed 22,400,000 lines: 0 bad lines : 69.66% complete\n",
      "Processed 22,400,000 lines: 0 bad lines : 69.66% complete\n",
      "Processed 22,500,000 lines: 0 bad lines : 69.99% complete\n",
      "Processed 22,500,000 lines: 0 bad lines : 69.99% complete\n",
      "Processed 22,500,000 lines: 0 bad lines : 69.99% complete\n",
      "Processed 22,600,000 lines: 0 bad lines : 70.22% complete\n",
      "Processed 22,600,000 lines: 0 bad lines : 70.22% complete\n",
      "Processed 22,600,000 lines: 0 bad lines : 70.22% complete\n",
      "Processed 22,700,000 lines: 0 bad lines : 70.55% complete\n",
      "Processed 22,700,000 lines: 0 bad lines : 70.55% complete\n",
      "Processed 22,700,000 lines: 0 bad lines : 70.55% complete\n",
      "Processed 22,800,000 lines: 0 bad lines : 70.87% complete\n",
      "Processed 22,800,000 lines: 0 bad lines : 70.87% complete\n",
      "Processed 22,800,000 lines: 0 bad lines : 70.87% complete\n",
      "Processed 22,900,000 lines: 0 bad lines : 71.19% complete\n",
      "Processed 22,900,000 lines: 0 bad lines : 71.19% complete\n",
      "Processed 22,900,000 lines: 0 bad lines : 71.19% complete\n",
      "Processed 23,000,000 lines: 0 bad lines : 71.51% complete\n",
      "Processed 23,000,000 lines: 0 bad lines : 71.51% complete\n",
      "Processed 23,000,000 lines: 0 bad lines : 71.51% complete\n",
      "Processed 23,100,000 lines: 0 bad lines : 71.83% complete\n",
      "Processed 23,100,000 lines: 0 bad lines : 71.83% complete\n",
      "Processed 23,100,000 lines: 0 bad lines : 71.83% complete\n",
      "Processed 23,200,000 lines: 0 bad lines : 72.05% complete\n",
      "Processed 23,200,000 lines: 0 bad lines : 72.05% complete\n",
      "Processed 23,200,000 lines: 0 bad lines : 72.05% complete\n",
      "Processed 23,300,000 lines: 0 bad lines : 72.39% complete\n",
      "Processed 23,300,000 lines: 0 bad lines : 72.39% complete\n",
      "Processed 23,300,000 lines: 0 bad lines : 72.39% complete\n",
      "Processed 23,400,000 lines: 0 bad lines : 72.72% complete\n",
      "Processed 23,400,000 lines: 0 bad lines : 72.72% complete\n",
      "Processed 23,400,000 lines: 0 bad lines : 72.72% complete\n",
      "Processed 23,500,000 lines: 0 bad lines : 73.05% complete\n",
      "Processed 23,500,000 lines: 0 bad lines : 73.05% complete\n",
      "Processed 23,500,000 lines: 0 bad lines : 73.05% complete\n",
      "Processed 23,600,000 lines: 0 bad lines : 73.38% complete\n",
      "Processed 23,600,000 lines: 0 bad lines : 73.38% complete\n",
      "Processed 23,600,000 lines: 0 bad lines : 73.38% complete\n",
      "Processed 23,700,000 lines: 0 bad lines : 73.70% complete\n",
      "Processed 23,700,000 lines: 0 bad lines : 73.70% complete\n",
      "Processed 23,700,000 lines: 0 bad lines : 73.70% complete\n",
      "Processed 23,800,000 lines: 0 bad lines : 73.92% complete\n",
      "Processed 23,800,000 lines: 0 bad lines : 73.92% complete\n",
      "Processed 23,800,000 lines: 0 bad lines : 73.92% complete\n",
      "Processed 23,900,000 lines: 0 bad lines : 74.23% complete\n",
      "Processed 23,900,000 lines: 0 bad lines : 74.23% complete\n",
      "Processed 23,900,000 lines: 0 bad lines : 74.23% complete\n",
      "Processed 24,000,000 lines: 0 bad lines : 74.54% complete\n",
      "Processed 24,000,000 lines: 0 bad lines : 74.54% complete\n",
      "Processed 24,000,000 lines: 0 bad lines : 74.54% complete\n",
      "Processed 24,100,000 lines: 0 bad lines : 74.86% complete\n",
      "Processed 24,100,000 lines: 0 bad lines : 74.86% complete\n",
      "Processed 24,100,000 lines: 0 bad lines : 74.86% complete\n",
      "Processed 24,200,000 lines: 0 bad lines : 75.19% complete\n",
      "Processed 24,200,000 lines: 0 bad lines : 75.19% complete\n",
      "Processed 24,200,000 lines: 0 bad lines : 75.19% complete\n",
      "Processed 24,300,000 lines: 0 bad lines : 75.52% complete\n",
      "Processed 24,300,000 lines: 0 bad lines : 75.52% complete\n",
      "Processed 24,300,000 lines: 0 bad lines : 75.52% complete\n",
      "Processed 24,400,000 lines: 0 bad lines : 75.85% complete\n",
      "Processed 24,400,000 lines: 0 bad lines : 75.85% complete\n",
      "Processed 24,400,000 lines: 0 bad lines : 75.85% complete\n",
      "Processed 24,500,000 lines: 0 bad lines : 76.18% complete\n",
      "Processed 24,500,000 lines: 0 bad lines : 76.18% complete\n",
      "Processed 24,500,000 lines: 0 bad lines : 76.18% complete\n",
      "Processed 24,600,000 lines: 0 bad lines : 76.40% complete\n",
      "Processed 24,600,000 lines: 0 bad lines : 76.40% complete\n",
      "Processed 24,600,000 lines: 0 bad lines : 76.40% complete\n",
      "Processed 24,700,000 lines: 0 bad lines : 76.73% complete\n",
      "Processed 24,700,000 lines: 0 bad lines : 76.73% complete\n",
      "Processed 24,700,000 lines: 0 bad lines : 76.73% complete\n",
      "Processed 24,800,000 lines: 0 bad lines : 77.04% complete\n",
      "Processed 24,800,000 lines: 0 bad lines : 77.04% complete\n",
      "Processed 24,800,000 lines: 0 bad lines : 77.04% complete\n",
      "Processed 24,900,000 lines: 0 bad lines : 77.36% complete\n",
      "Processed 24,900,000 lines: 0 bad lines : 77.36% complete\n",
      "Processed 24,900,000 lines: 0 bad lines : 77.36% complete\n",
      "Processed 25,000,000 lines: 0 bad lines : 77.68% complete\n",
      "Processed 25,000,000 lines: 0 bad lines : 77.68% complete\n",
      "Processed 25,000,000 lines: 0 bad lines : 77.68% complete\n",
      "Processed 25,100,000 lines: 0 bad lines : 78.01% complete\n",
      "Processed 25,100,000 lines: 0 bad lines : 78.01% complete\n",
      "Processed 25,100,000 lines: 0 bad lines : 78.01% complete\n",
      "Processed 25,200,000 lines: 0 bad lines : 78.23% complete\n",
      "Processed 25,200,000 lines: 0 bad lines : 78.23% complete\n",
      "Processed 25,200,000 lines: 0 bad lines : 78.23% complete\n",
      "Processed 25,300,000 lines: 0 bad lines : 78.55% complete\n",
      "Processed 25,300,000 lines: 0 bad lines : 78.55% complete\n",
      "Processed 25,300,000 lines: 0 bad lines : 78.55% complete\n",
      "Processed 25,400,000 lines: 0 bad lines : 78.88% complete\n",
      "Processed 25,400,000 lines: 0 bad lines : 78.88% complete\n",
      "Processed 25,400,000 lines: 0 bad lines : 78.88% complete\n",
      "Processed 25,500,000 lines: 0 bad lines : 79.21% complete\n",
      "Processed 25,500,000 lines: 0 bad lines : 79.21% complete\n",
      "Processed 25,500,000 lines: 0 bad lines : 79.21% complete\n",
      "Processed 25,600,000 lines: 0 bad lines : 79.54% complete\n",
      "Processed 25,600,000 lines: 0 bad lines : 79.54% complete\n",
      "Processed 25,600,000 lines: 0 bad lines : 79.54% complete\n",
      "Processed 25,700,000 lines: 0 bad lines : 79.76% complete\n",
      "Processed 25,700,000 lines: 0 bad lines : 79.76% complete\n",
      "Processed 25,700,000 lines: 0 bad lines : 79.76% complete\n",
      "Processed 25,800,000 lines: 0 bad lines : 80.08% complete\n",
      "Processed 25,800,000 lines: 0 bad lines : 80.08% complete\n",
      "Processed 25,800,000 lines: 0 bad lines : 80.08% complete\n",
      "Processed 25,900,000 lines: 0 bad lines : 80.39% complete\n",
      "Processed 25,900,000 lines: 0 bad lines : 80.39% complete\n",
      "Processed 25,900,000 lines: 0 bad lines : 80.39% complete\n",
      "Processed 26,000,000 lines: 0 bad lines : 80.70% complete\n",
      "Processed 26,000,000 lines: 0 bad lines : 80.70% complete\n",
      "Processed 26,000,000 lines: 0 bad lines : 80.70% complete\n",
      "Processed 26,100,000 lines: 0 bad lines : 81.02% complete\n",
      "Processed 26,100,000 lines: 0 bad lines : 81.02% complete\n",
      "Processed 26,100,000 lines: 0 bad lines : 81.02% complete\n",
      "Processed 26,200,000 lines: 0 bad lines : 81.34% complete\n",
      "Processed 26,200,000 lines: 0 bad lines : 81.34% complete\n",
      "Processed 26,200,000 lines: 0 bad lines : 81.34% complete\n",
      "Processed 26,300,000 lines: 0 bad lines : 81.67% complete\n",
      "Processed 26,300,000 lines: 0 bad lines : 81.67% complete\n",
      "Processed 26,300,000 lines: 0 bad lines : 81.67% complete\n",
      "Processed 26,400,000 lines: 0 bad lines : 81.89% complete\n",
      "Processed 26,400,000 lines: 0 bad lines : 81.89% complete\n",
      "Processed 26,400,000 lines: 0 bad lines : 81.89% complete\n",
      "Processed 26,500,000 lines: 0 bad lines : 82.22% complete\n",
      "Processed 26,500,000 lines: 0 bad lines : 82.22% complete\n",
      "Processed 26,500,000 lines: 0 bad lines : 82.22% complete\n",
      "Processed 26,600,000 lines: 0 bad lines : 82.55% complete\n",
      "Processed 26,600,000 lines: 0 bad lines : 82.55% complete\n",
      "Processed 26,600,000 lines: 0 bad lines : 82.55% complete\n",
      "Processed 26,700,000 lines: 0 bad lines : 82.88% complete\n",
      "Processed 26,700,000 lines: 0 bad lines : 82.88% complete\n",
      "Processed 26,700,000 lines: 0 bad lines : 82.88% complete\n",
      "Processed 26,800,000 lines: 0 bad lines : 83.21% complete\n",
      "Processed 26,800,000 lines: 0 bad lines : 83.21% complete\n",
      "Processed 26,800,000 lines: 0 bad lines : 83.21% complete\n",
      "Processed 26,900,000 lines: 0 bad lines : 83.43% complete\n",
      "Processed 26,900,000 lines: 0 bad lines : 83.43% complete\n",
      "Processed 26,900,000 lines: 0 bad lines : 83.43% complete\n",
      "Processed 27,000,000 lines: 0 bad lines : 83.74% complete\n",
      "Processed 27,000,000 lines: 0 bad lines : 83.74% complete\n",
      "Processed 27,000,000 lines: 0 bad lines : 83.74% complete\n",
      "Processed 27,100,000 lines: 0 bad lines : 84.05% complete\n",
      "Processed 27,100,000 lines: 0 bad lines : 84.05% complete\n",
      "Processed 27,100,000 lines: 0 bad lines : 84.05% complete\n",
      "Processed 27,200,000 lines: 0 bad lines : 84.36% complete\n",
      "Processed 27,200,000 lines: 0 bad lines : 84.36% complete\n",
      "Processed 27,200,000 lines: 0 bad lines : 84.36% complete\n",
      "Processed 27,300,000 lines: 0 bad lines : 84.68% complete\n",
      "Processed 27,300,000 lines: 0 bad lines : 84.68% complete\n",
      "Processed 27,300,000 lines: 0 bad lines : 84.68% complete\n",
      "Processed 27,400,000 lines: 0 bad lines : 85.00% complete\n",
      "Processed 27,400,000 lines: 0 bad lines : 85.00% complete\n",
      "Processed 27,400,000 lines: 0 bad lines : 85.00% complete\n",
      "Processed 27,500,000 lines: 0 bad lines : 85.22% complete\n",
      "Processed 27,500,000 lines: 0 bad lines : 85.22% complete\n",
      "Processed 27,500,000 lines: 0 bad lines : 85.22% complete\n",
      "Processed 27,600,000 lines: 0 bad lines : 85.54% complete\n",
      "Processed 27,600,000 lines: 0 bad lines : 85.54% complete\n",
      "Processed 27,600,000 lines: 0 bad lines : 85.54% complete\n",
      "Processed 27,700,000 lines: 0 bad lines : 85.85% complete\n",
      "Processed 27,700,000 lines: 0 bad lines : 85.85% complete\n",
      "Processed 27,700,000 lines: 0 bad lines : 85.85% complete\n",
      "Processed 27,800,000 lines: 0 bad lines : 86.07% complete\n",
      "Processed 27,800,000 lines: 0 bad lines : 86.07% complete\n",
      "Processed 27,800,000 lines: 0 bad lines : 86.07% complete\n",
      "Processed 27,900,000 lines: 0 bad lines : 86.37% complete\n",
      "Processed 27,900,000 lines: 0 bad lines : 86.37% complete\n",
      "Processed 27,900,000 lines: 0 bad lines : 86.37% complete\n",
      "Processed 28,000,000 lines: 0 bad lines : 86.69% complete\n",
      "Processed 28,000,000 lines: 0 bad lines : 86.69% complete\n",
      "Processed 28,000,000 lines: 0 bad lines : 86.69% complete\n",
      "Processed 28,100,000 lines: 0 bad lines : 87.00% complete\n",
      "Processed 28,100,000 lines: 0 bad lines : 87.00% complete\n",
      "Processed 28,100,000 lines: 0 bad lines : 87.00% complete\n",
      "Processed 28,200,000 lines: 0 bad lines : 87.20% complete\n",
      "Processed 28,200,000 lines: 0 bad lines : 87.20% complete\n",
      "Processed 28,200,000 lines: 0 bad lines : 87.20% complete\n",
      "Processed 28,300,000 lines: 0 bad lines : 87.50% complete\n",
      "Processed 28,300,000 lines: 0 bad lines : 87.50% complete\n",
      "Processed 28,300,000 lines: 0 bad lines : 87.50% complete\n",
      "Processed 28,400,000 lines: 0 bad lines : 87.80% complete\n",
      "Processed 28,400,000 lines: 0 bad lines : 87.80% complete\n",
      "Processed 28,400,000 lines: 0 bad lines : 87.80% complete\n",
      "Processed 28,500,000 lines: 0 bad lines : 88.09% complete\n",
      "Processed 28,500,000 lines: 0 bad lines : 88.09% complete\n",
      "Processed 28,500,000 lines: 0 bad lines : 88.09% complete\n",
      "Processed 28,600,000 lines: 0 bad lines : 88.29% complete\n",
      "Processed 28,600,000 lines: 0 bad lines : 88.29% complete\n",
      "Processed 28,600,000 lines: 0 bad lines : 88.29% complete\n",
      "Processed 28,700,000 lines: 0 bad lines : 88.60% complete\n",
      "Processed 28,700,000 lines: 0 bad lines : 88.60% complete\n",
      "Processed 28,700,000 lines: 0 bad lines : 88.60% complete\n",
      "Processed 28,800,000 lines: 0 bad lines : 88.80% complete\n",
      "Processed 28,800,000 lines: 0 bad lines : 88.80% complete\n",
      "Processed 28,800,000 lines: 0 bad lines : 88.80% complete\n",
      "Processed 28,900,000 lines: 0 bad lines : 89.10% complete\n",
      "Processed 28,900,000 lines: 0 bad lines : 89.10% complete\n",
      "Processed 28,900,000 lines: 0 bad lines : 89.10% complete\n",
      "Processed 29,000,000 lines: 0 bad lines : 89.41% complete\n",
      "Processed 29,000,000 lines: 0 bad lines : 89.41% complete\n",
      "Processed 29,000,000 lines: 0 bad lines : 89.41% complete\n",
      "Processed 29,100,000 lines: 0 bad lines : 89.71% complete\n",
      "Processed 29,100,000 lines: 0 bad lines : 89.71% complete\n",
      "Processed 29,100,000 lines: 0 bad lines : 89.71% complete\n",
      "Processed 29,200,000 lines: 0 bad lines : 89.91% complete\n",
      "Processed 29,200,000 lines: 0 bad lines : 89.91% complete\n",
      "Processed 29,200,000 lines: 0 bad lines : 89.91% complete\n",
      "Processed 29,300,000 lines: 0 bad lines : 90.21% complete\n",
      "Processed 29,300,000 lines: 0 bad lines : 90.21% complete\n",
      "Processed 29,300,000 lines: 0 bad lines : 90.21% complete\n",
      "Processed 29,400,000 lines: 0 bad lines : 90.50% complete\n",
      "Processed 29,400,000 lines: 0 bad lines : 90.50% complete\n",
      "Processed 29,400,000 lines: 0 bad lines : 90.50% complete\n",
      "Processed 29,500,000 lines: 0 bad lines : 90.80% complete\n",
      "Processed 29,500,000 lines: 0 bad lines : 90.80% complete\n",
      "Processed 29,500,000 lines: 0 bad lines : 90.80% complete\n",
      "Processed 29,600,000 lines: 0 bad lines : 91.10% complete\n",
      "Processed 29,600,000 lines: 0 bad lines : 91.10% complete\n",
      "Processed 29,600,000 lines: 0 bad lines : 91.10% complete\n",
      "Processed 29,700,000 lines: 0 bad lines : 91.30% complete\n",
      "Processed 29,700,000 lines: 0 bad lines : 91.30% complete\n",
      "Processed 29,700,000 lines: 0 bad lines : 91.30% complete\n",
      "Processed 29,800,000 lines: 0 bad lines : 91.60% complete\n",
      "Processed 29,800,000 lines: 0 bad lines : 91.60% complete\n",
      "Processed 29,800,000 lines: 0 bad lines : 91.60% complete\n",
      "Processed 29,900,000 lines: 0 bad lines : 91.90% complete\n",
      "Processed 29,900,000 lines: 0 bad lines : 91.90% complete\n",
      "Processed 29,900,000 lines: 0 bad lines : 91.90% complete\n",
      "Processed 30,000,000 lines: 0 bad lines : 92.21% complete\n",
      "Processed 30,000,000 lines: 0 bad lines : 92.21% complete\n",
      "Processed 30,000,000 lines: 0 bad lines : 92.21% complete\n",
      "Processed 30,100,000 lines: 0 bad lines : 92.42% complete\n",
      "Processed 30,100,000 lines: 0 bad lines : 92.42% complete\n",
      "Processed 30,100,000 lines: 0 bad lines : 92.42% complete\n",
      "Processed 30,200,000 lines: 0 bad lines : 92.73% complete\n",
      "Processed 30,200,000 lines: 0 bad lines : 92.73% complete\n",
      "Processed 30,200,000 lines: 0 bad lines : 92.73% complete\n",
      "Processed 30,300,000 lines: 0 bad lines : 93.03% complete\n",
      "Processed 30,300,000 lines: 0 bad lines : 93.03% complete\n",
      "Processed 30,300,000 lines: 0 bad lines : 93.03% complete\n",
      "Processed 30,400,000 lines: 0 bad lines : 93.34% complete\n",
      "Processed 30,400,000 lines: 0 bad lines : 93.34% complete\n",
      "Processed 30,400,000 lines: 0 bad lines : 93.34% complete\n",
      "Processed 30,500,000 lines: 0 bad lines : 93.55% complete\n",
      "Processed 30,500,000 lines: 0 bad lines : 93.55% complete\n",
      "Processed 30,500,000 lines: 0 bad lines : 93.55% complete\n",
      "Processed 30,600,000 lines: 0 bad lines : 93.86% complete\n",
      "Processed 30,600,000 lines: 0 bad lines : 93.86% complete\n",
      "Processed 30,600,000 lines: 0 bad lines : 93.86% complete\n",
      "Processed 30,700,000 lines: 0 bad lines : 94.18% complete\n",
      "Processed 30,700,000 lines: 0 bad lines : 94.18% complete\n",
      "Processed 30,700,000 lines: 0 bad lines : 94.18% complete\n",
      "Processed 30,800,000 lines: 0 bad lines : 94.49% complete\n",
      "Processed 30,800,000 lines: 0 bad lines : 94.49% complete\n",
      "Processed 30,800,000 lines: 0 bad lines : 94.49% complete\n",
      "Processed 30,900,000 lines: 0 bad lines : 94.69% complete\n",
      "Processed 30,900,000 lines: 0 bad lines : 94.69% complete\n",
      "Processed 30,900,000 lines: 0 bad lines : 94.69% complete\n",
      "Processed 31,000,000 lines: 0 bad lines : 94.99% complete\n",
      "Processed 31,000,000 lines: 0 bad lines : 94.99% complete\n",
      "Processed 31,000,000 lines: 0 bad lines : 94.99% complete\n",
      "Processed 31,100,000 lines: 0 bad lines : 95.29% complete\n",
      "Processed 31,100,000 lines: 0 bad lines : 95.29% complete\n",
      "Processed 31,100,000 lines: 0 bad lines : 95.29% complete\n",
      "Processed 31,200,000 lines: 0 bad lines : 95.61% complete\n",
      "Processed 31,200,000 lines: 0 bad lines : 95.61% complete\n",
      "Processed 31,200,000 lines: 0 bad lines : 95.61% complete\n",
      "Processed 31,300,000 lines: 0 bad lines : 95.82% complete\n",
      "Processed 31,300,000 lines: 0 bad lines : 95.82% complete\n",
      "Processed 31,300,000 lines: 0 bad lines : 95.82% complete\n",
      "Processed 31,400,000 lines: 0 bad lines : 96.13% complete\n",
      "Processed 31,400,000 lines: 0 bad lines : 96.13% complete\n",
      "Processed 31,400,000 lines: 0 bad lines : 96.13% complete\n",
      "Processed 31,500,000 lines: 0 bad lines : 96.44% complete\n",
      "Processed 31,500,000 lines: 0 bad lines : 96.44% complete\n",
      "Processed 31,500,000 lines: 0 bad lines : 96.44% complete\n",
      "Processed 31,600,000 lines: 0 bad lines : 96.76% complete\n",
      "Processed 31,600,000 lines: 0 bad lines : 96.76% complete\n",
      "Processed 31,600,000 lines: 0 bad lines : 96.76% complete\n",
      "Processed 31,700,000 lines: 0 bad lines : 97.07% complete\n",
      "Processed 31,700,000 lines: 0 bad lines : 97.07% complete\n",
      "Processed 31,700,000 lines: 0 bad lines : 97.07% complete\n",
      "Processed 31,800,000 lines: 0 bad lines : 97.37% complete\n",
      "Processed 31,800,000 lines: 0 bad lines : 97.37% complete\n",
      "Processed 31,800,000 lines: 0 bad lines : 97.37% complete\n",
      "Processed 31,900,000 lines: 0 bad lines : 97.68% complete\n",
      "Processed 31,900,000 lines: 0 bad lines : 97.68% complete\n",
      "Processed 31,900,000 lines: 0 bad lines : 97.68% complete\n",
      "Processed 32,000,000 lines: 0 bad lines : 97.87% complete\n",
      "Processed 32,000,000 lines: 0 bad lines : 97.87% complete\n",
      "Processed 32,000,000 lines: 0 bad lines : 97.87% complete\n",
      "Processed 32,100,000 lines: 0 bad lines : 98.17% complete\n",
      "Processed 32,100,000 lines: 0 bad lines : 98.17% complete\n",
      "Processed 32,100,000 lines: 0 bad lines : 98.17% complete\n",
      "Processed 32,200,000 lines: 0 bad lines : 98.47% complete\n",
      "Processed 32,200,000 lines: 0 bad lines : 98.47% complete\n",
      "Processed 32,200,000 lines: 0 bad lines : 98.47% complete\n",
      "Processed 32,300,000 lines: 0 bad lines : 98.77% complete\n",
      "Processed 32,300,000 lines: 0 bad lines : 98.77% complete\n",
      "Processed 32,300,000 lines: 0 bad lines : 98.77% complete\n",
      "Processed 32,400,000 lines: 0 bad lines : 99.09% complete\n",
      "Processed 32,400,000 lines: 0 bad lines : 99.09% complete\n",
      "Processed 32,400,000 lines: 0 bad lines : 99.09% complete\n",
      "Processed 32,500,000 lines: 0 bad lines : 99.41% complete\n",
      "Processed 32,500,000 lines: 0 bad lines : 99.41% complete\n",
      "Processed 32,500,000 lines: 0 bad lines : 99.41% complete\n",
      "Processed 32,600,000 lines: 0 bad lines : 99.74% complete\n",
      "Processed 32,600,000 lines: 0 bad lines : 99.74% complete\n",
      "Processed 32,600,000 lines: 0 bad lines : 99.74% complete\n",
      "Processed 32,700,000 lines: 0 bad lines : 100.00% complete\n",
      "Processed 32,700,000 lines: 0 bad lines : 100.00% complete\n",
      "Processed 32,700,000 lines: 0 bad lines : 100.00% complete\n",
      "Complete : 32,704,571 lines processed with 0 bad lines.\n",
      "Complete : 32,704,571 lines processed with 0 bad lines.\n",
      "Complete : 32,704,571 lines processed with 0 bad lines.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "log = logging.getLogger(\"bot\")\n",
    "log.setLevel(logging.DEBUG)\n",
    "log.addHandler(logging.StreamHandler())\n",
    "\n",
    "def read_and_decode(reader, chunk_size, max_window_size, previous_chunk=None, bytes_read=0):\n",
    "    chunk = reader.read(chunk_size)\n",
    "    bytes_read += chunk_size\n",
    "    if previous_chunk is not None:\n",
    "        chunk = previous_chunk + chunk\n",
    "    try:\n",
    "        return chunk.decode()\n",
    "    except UnicodeDecodeError:\n",
    "        if bytes_read > max_window_size:\n",
    "            raise UnicodeError(f\"Unable to decode frame after reading {bytes_read:,} bytes\")\n",
    "        return read_and_decode(reader, chunk_size, max_window_size, chunk, bytes_read)\n",
    "\n",
    "def read_lines_zst(file_name):\n",
    "    with open(file_name, 'rb') as file_handle:\n",
    "        buffer = ''\n",
    "        reader = zstandard.ZstdDecompressor(max_window_size=2**31).stream_reader(file_handle)\n",
    "        while True:\n",
    "            chunk = read_and_decode(reader, 2**27, (2**29) * 2)\n",
    "            if not chunk:\n",
    "                break\n",
    "            lines = (buffer + chunk).split(\"\\n\")\n",
    "            for line in lines[:-1]:\n",
    "                yield line, file_handle.tell()\n",
    "            buffer = lines[-1]\n",
    "        reader.close()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    zst_file_name = 'RS_2021-01'\n",
    "    input_file_path = '../../submissions/' + zst_file_name + '.zst'\n",
    "    output_file_path = '../submissions/csv/2021/'+ zst_file_name +'.csv'\n",
    "\n",
    "    # List of subreddits to filter by\n",
    "    subreddits = ['NOLA', 'acadiana', 'centrallouisiana', 'houma', 'louisiana', \n",
    "                  'batonrouge', 'neworleans', 'Thibodaux', 'newyork', 'Adirondacks', \n",
    "                  'bronx', 'Brooklyn', 'Bushwick', 'hudsonvalley', 'OrangeCountyNY', \n",
    "                  'Queens', 'Westchester', 'Buffalo', 'ForestHills', 'longisland', \n",
    "                  'nyc', 'parkslope', 'williamsburg', 'newjersey', 'Hoboken', \n",
    "                  'jerseycity', 'Newark', 'SouthJersey', 'asburypark', 'NewBrunswickNJ', \n",
    "                  'texas', 'Austin', 'CorpusChristi', 'galveston', 'pflugerville', \n",
    "                  'sanantonio', 'corpus', 'Denton', 'houston', 'TriCitiesWA', 'Washington', 'oregon', 'Seattle', 'SeattleWA', 'Portland']\n",
    "\n",
    "    fields = ['id', 'author', 'title', 'subreddit', 'created_utc', 'url', 'num_comments', 'score', 'selftext']\n",
    "\n",
    "    file_size = os.stat(input_file_path).st_size\n",
    "    file_lines = 0\n",
    "    file_bytes_processed = 0\n",
    "    bad_lines = 0\n",
    "\n",
    "    with open(output_file_path, \"w\", encoding='utf-8', newline=\"\") as output_file:\n",
    "        writer = csv.writer(output_file)\n",
    "        writer.writerow(fields)  # Write the header\n",
    "\n",
    "        for line, file_bytes_processed in read_lines_zst(input_file_path):\n",
    "            try:\n",
    "                obj = json.loads(line)\n",
    "                if obj.get('subreddit', '').lower() in (sub.lower() for sub in subreddits):\n",
    "                    writer.writerow([str(obj.get(field, '')).replace(\"\\n\", \" \") for field in fields])\n",
    "            except json.JSONDecodeError as err:\n",
    "                bad_lines += 1\n",
    "            except KeyError as err:\n",
    "                log.info(f\"Missing field in line: {err}\")\n",
    "                bad_lines += 1\n",
    "\n",
    "            file_lines += 1\n",
    "            if file_lines % 100000 == 0:\n",
    "                log.info(f\"Processed {file_lines:,} lines: {bad_lines:,} bad lines : {(file_bytes_processed / file_size) * 100:.2f}% complete\")\n",
    "\n",
    "    log.info(f\"Complete : {file_lines:,} lines processed with {bad_lines:,} bad lines.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processed 100,000 lines: 0 bad lines : 0.09% complete\n",
      "Processed 100,000 lines: 0 bad lines : 0.09% complete\n",
      "Processed 200,000 lines: 0 bad lines : 0.13% complete\n",
      "Processed 200,000 lines: 0 bad lines : 0.13% complete\n",
      "Processed 300,000 lines: 0 bad lines : 0.18% complete\n",
      "Processed 300,000 lines: 0 bad lines : 0.18% complete\n",
      "Processed 400,000 lines: 0 bad lines : 0.22% complete\n",
      "Processed 400,000 lines: 0 bad lines : 0.22% complete\n",
      "Processed 500,000 lines: 0 bad lines : 0.26% complete\n",
      "Processed 500,000 lines: 0 bad lines : 0.26% complete\n",
      "Processed 600,000 lines: 0 bad lines : 0.30% complete\n",
      "Processed 600,000 lines: 0 bad lines : 0.30% complete\n",
      "Processed 700,000 lines: 0 bad lines : 0.34% complete\n",
      "Processed 700,000 lines: 0 bad lines : 0.34% complete\n",
      "Processed 800,000 lines: 0 bad lines : 0.39% complete\n",
      "Processed 800,000 lines: 0 bad lines : 0.39% complete\n",
      "Processed 900,000 lines: 0 bad lines : 0.43% complete\n",
      "Processed 900,000 lines: 0 bad lines : 0.43% complete\n",
      "Processed 1,000,000 lines: 0 bad lines : 0.47% complete\n",
      "Processed 1,000,000 lines: 0 bad lines : 0.47% complete\n",
      "Processed 1,100,000 lines: 0 bad lines : 0.51% complete\n",
      "Processed 1,100,000 lines: 0 bad lines : 0.51% complete\n",
      "Processed 1,200,000 lines: 0 bad lines : 0.55% complete\n",
      "Processed 1,200,000 lines: 0 bad lines : 0.55% complete\n",
      "Processed 1,300,000 lines: 0 bad lines : 0.60% complete\n",
      "Processed 1,300,000 lines: 0 bad lines : 0.60% complete\n",
      "Processed 1,400,000 lines: 0 bad lines : 0.64% complete\n",
      "Processed 1,400,000 lines: 0 bad lines : 0.64% complete\n",
      "Processed 1,500,000 lines: 0 bad lines : 0.68% complete\n",
      "Processed 1,500,000 lines: 0 bad lines : 0.68% complete\n",
      "Processed 1,600,000 lines: 0 bad lines : 0.72% complete\n",
      "Processed 1,600,000 lines: 0 bad lines : 0.72% complete\n",
      "Processed 1,700,000 lines: 0 bad lines : 0.76% complete\n",
      "Processed 1,700,000 lines: 0 bad lines : 0.76% complete\n",
      "Processed 1,800,000 lines: 0 bad lines : 0.80% complete\n",
      "Processed 1,800,000 lines: 0 bad lines : 0.80% complete\n",
      "Processed 1,900,000 lines: 0 bad lines : 0.84% complete\n",
      "Processed 1,900,000 lines: 0 bad lines : 0.84% complete\n",
      "Processed 2,000,000 lines: 0 bad lines : 0.89% complete\n",
      "Processed 2,000,000 lines: 0 bad lines : 0.89% complete\n",
      "Processed 2,100,000 lines: 0 bad lines : 0.93% complete\n",
      "Processed 2,100,000 lines: 0 bad lines : 0.93% complete\n",
      "Processed 2,200,000 lines: 0 bad lines : 0.97% complete\n",
      "Processed 2,200,000 lines: 0 bad lines : 0.97% complete\n",
      "Processed 2,300,000 lines: 0 bad lines : 1.01% complete\n",
      "Processed 2,300,000 lines: 0 bad lines : 1.01% complete\n",
      "Processed 2,400,000 lines: 0 bad lines : 1.05% complete\n",
      "Processed 2,400,000 lines: 0 bad lines : 1.05% complete\n",
      "Processed 2,500,000 lines: 0 bad lines : 1.09% complete\n",
      "Processed 2,500,000 lines: 0 bad lines : 1.09% complete\n",
      "Processed 2,600,000 lines: 0 bad lines : 1.13% complete\n",
      "Processed 2,600,000 lines: 0 bad lines : 1.13% complete\n",
      "Processed 2,700,000 lines: 0 bad lines : 1.18% complete\n",
      "Processed 2,700,000 lines: 0 bad lines : 1.18% complete\n",
      "Processed 2,800,000 lines: 0 bad lines : 1.22% complete\n",
      "Processed 2,800,000 lines: 0 bad lines : 1.22% complete\n",
      "Processed 2,900,000 lines: 0 bad lines : 1.26% complete\n",
      "Processed 2,900,000 lines: 0 bad lines : 1.26% complete\n",
      "Processed 3,000,000 lines: 0 bad lines : 1.30% complete\n",
      "Processed 3,000,000 lines: 0 bad lines : 1.30% complete\n",
      "Processed 3,100,000 lines: 0 bad lines : 1.34% complete\n",
      "Processed 3,100,000 lines: 0 bad lines : 1.34% complete\n",
      "Processed 3,200,000 lines: 0 bad lines : 1.38% complete\n",
      "Processed 3,200,000 lines: 0 bad lines : 1.38% complete\n",
      "Processed 3,300,000 lines: 0 bad lines : 1.43% complete\n",
      "Processed 3,300,000 lines: 0 bad lines : 1.43% complete\n",
      "Processed 3,400,000 lines: 0 bad lines : 1.51% complete\n",
      "Processed 3,400,000 lines: 0 bad lines : 1.51% complete\n",
      "Processed 3,500,000 lines: 0 bad lines : 1.55% complete\n",
      "Processed 3,500,000 lines: 0 bad lines : 1.55% complete\n",
      "Processed 3,600,000 lines: 0 bad lines : 1.59% complete\n",
      "Processed 3,600,000 lines: 0 bad lines : 1.59% complete\n",
      "Processed 3,700,000 lines: 0 bad lines : 1.64% complete\n",
      "Processed 3,700,000 lines: 0 bad lines : 1.64% complete\n",
      "Processed 3,800,000 lines: 0 bad lines : 1.68% complete\n",
      "Processed 3,800,000 lines: 0 bad lines : 1.68% complete\n",
      "Processed 3,900,000 lines: 0 bad lines : 1.72% complete\n",
      "Processed 3,900,000 lines: 0 bad lines : 1.72% complete\n",
      "Processed 4,000,000 lines: 0 bad lines : 1.77% complete\n",
      "Processed 4,000,000 lines: 0 bad lines : 1.77% complete\n",
      "Processed 4,100,000 lines: 0 bad lines : 1.81% complete\n",
      "Processed 4,100,000 lines: 0 bad lines : 1.81% complete\n",
      "Processed 4,200,000 lines: 0 bad lines : 1.85% complete\n",
      "Processed 4,200,000 lines: 0 bad lines : 1.85% complete\n",
      "Processed 4,300,000 lines: 0 bad lines : 1.90% complete\n",
      "Processed 4,300,000 lines: 0 bad lines : 1.90% complete\n",
      "Processed 4,400,000 lines: 0 bad lines : 1.94% complete\n",
      "Processed 4,400,000 lines: 0 bad lines : 1.94% complete\n",
      "Processed 4,500,000 lines: 0 bad lines : 1.98% complete\n",
      "Processed 4,500,000 lines: 0 bad lines : 1.98% complete\n",
      "Processed 4,600,000 lines: 0 bad lines : 2.03% complete\n",
      "Processed 4,600,000 lines: 0 bad lines : 2.03% complete\n",
      "Processed 4,700,000 lines: 0 bad lines : 2.07% complete\n",
      "Processed 4,700,000 lines: 0 bad lines : 2.07% complete\n",
      "Processed 4,800,000 lines: 0 bad lines : 2.11% complete\n",
      "Processed 4,800,000 lines: 0 bad lines : 2.11% complete\n",
      "Processed 4,900,000 lines: 0 bad lines : 2.15% complete\n",
      "Processed 4,900,000 lines: 0 bad lines : 2.15% complete\n",
      "Processed 5,000,000 lines: 0 bad lines : 2.20% complete\n",
      "Processed 5,000,000 lines: 0 bad lines : 2.20% complete\n",
      "Processed 5,100,000 lines: 0 bad lines : 2.24% complete\n",
      "Processed 5,100,000 lines: 0 bad lines : 2.24% complete\n",
      "Processed 5,200,000 lines: 0 bad lines : 2.28% complete\n",
      "Processed 5,200,000 lines: 0 bad lines : 2.28% complete\n",
      "Processed 5,300,000 lines: 0 bad lines : 2.33% complete\n",
      "Processed 5,300,000 lines: 0 bad lines : 2.33% complete\n",
      "Processed 5,400,000 lines: 0 bad lines : 2.37% complete\n",
      "Processed 5,400,000 lines: 0 bad lines : 2.37% complete\n",
      "Processed 5,500,000 lines: 0 bad lines : 2.41% complete\n",
      "Processed 5,500,000 lines: 0 bad lines : 2.41% complete\n",
      "Processed 5,600,000 lines: 0 bad lines : 2.45% complete\n",
      "Processed 5,600,000 lines: 0 bad lines : 2.45% complete\n",
      "Processed 5,700,000 lines: 0 bad lines : 2.50% complete\n",
      "Processed 5,700,000 lines: 0 bad lines : 2.50% complete\n",
      "Processed 5,800,000 lines: 0 bad lines : 2.54% complete\n",
      "Processed 5,800,000 lines: 0 bad lines : 2.54% complete\n",
      "Processed 5,900,000 lines: 0 bad lines : 2.58% complete\n",
      "Processed 5,900,000 lines: 0 bad lines : 2.58% complete\n",
      "Processed 6,000,000 lines: 0 bad lines : 2.67% complete\n",
      "Processed 6,000,000 lines: 0 bad lines : 2.67% complete\n",
      "Processed 6,100,000 lines: 0 bad lines : 2.71% complete\n",
      "Processed 6,100,000 lines: 0 bad lines : 2.71% complete\n",
      "Processed 6,200,000 lines: 0 bad lines : 2.75% complete\n",
      "Processed 6,200,000 lines: 0 bad lines : 2.75% complete\n",
      "Processed 6,300,000 lines: 0 bad lines : 2.80% complete\n",
      "Processed 6,300,000 lines: 0 bad lines : 2.80% complete\n",
      "Processed 6,400,000 lines: 0 bad lines : 2.84% complete\n",
      "Processed 6,400,000 lines: 0 bad lines : 2.84% complete\n",
      "Processed 6,500,000 lines: 0 bad lines : 2.88% complete\n",
      "Processed 6,500,000 lines: 0 bad lines : 2.88% complete\n",
      "Processed 6,600,000 lines: 0 bad lines : 2.92% complete\n",
      "Processed 6,600,000 lines: 0 bad lines : 2.92% complete\n",
      "Processed 6,700,000 lines: 0 bad lines : 2.97% complete\n",
      "Processed 6,700,000 lines: 0 bad lines : 2.97% complete\n",
      "Processed 6,800,000 lines: 0 bad lines : 3.01% complete\n",
      "Processed 6,800,000 lines: 0 bad lines : 3.01% complete\n",
      "Processed 6,900,000 lines: 0 bad lines : 3.05% complete\n",
      "Processed 6,900,000 lines: 0 bad lines : 3.05% complete\n",
      "Processed 7,000,000 lines: 0 bad lines : 3.09% complete\n",
      "Processed 7,000,000 lines: 0 bad lines : 3.09% complete\n",
      "Processed 7,100,000 lines: 0 bad lines : 3.14% complete\n",
      "Processed 7,100,000 lines: 0 bad lines : 3.14% complete\n",
      "Processed 7,200,000 lines: 0 bad lines : 3.18% complete\n",
      "Processed 7,200,000 lines: 0 bad lines : 3.18% complete\n",
      "Processed 7,300,000 lines: 0 bad lines : 3.22% complete\n",
      "Processed 7,300,000 lines: 0 bad lines : 3.22% complete\n",
      "Processed 7,400,000 lines: 0 bad lines : 3.26% complete\n",
      "Processed 7,400,000 lines: 0 bad lines : 3.26% complete\n",
      "Processed 7,500,000 lines: 0 bad lines : 3.30% complete\n",
      "Processed 7,500,000 lines: 0 bad lines : 3.30% complete\n",
      "Processed 7,600,000 lines: 0 bad lines : 3.34% complete\n",
      "Processed 7,600,000 lines: 0 bad lines : 3.34% complete\n",
      "Processed 7,700,000 lines: 0 bad lines : 3.39% complete\n",
      "Processed 7,700,000 lines: 0 bad lines : 3.39% complete\n",
      "Processed 7,800,000 lines: 0 bad lines : 3.43% complete\n",
      "Processed 7,800,000 lines: 0 bad lines : 3.43% complete\n",
      "Processed 7,900,000 lines: 0 bad lines : 3.47% complete\n",
      "Processed 7,900,000 lines: 0 bad lines : 3.47% complete\n",
      "Processed 8,000,000 lines: 0 bad lines : 3.51% complete\n",
      "Processed 8,000,000 lines: 0 bad lines : 3.51% complete\n",
      "Processed 8,100,000 lines: 0 bad lines : 3.55% complete\n",
      "Processed 8,100,000 lines: 0 bad lines : 3.55% complete\n",
      "Processed 8,200,000 lines: 0 bad lines : 3.60% complete\n",
      "Processed 8,200,000 lines: 0 bad lines : 3.60% complete\n",
      "Processed 8,300,000 lines: 0 bad lines : 3.64% complete\n",
      "Processed 8,300,000 lines: 0 bad lines : 3.64% complete\n",
      "Processed 8,400,000 lines: 0 bad lines : 3.68% complete\n",
      "Processed 8,400,000 lines: 0 bad lines : 3.68% complete\n",
      "Processed 8,500,000 lines: 0 bad lines : 3.72% complete\n",
      "Processed 8,500,000 lines: 0 bad lines : 3.72% complete\n",
      "Processed 8,600,000 lines: 0 bad lines : 3.77% complete\n",
      "Processed 8,600,000 lines: 0 bad lines : 3.77% complete\n",
      "Processed 8,700,000 lines: 0 bad lines : 3.81% complete\n",
      "Processed 8,700,000 lines: 0 bad lines : 3.81% complete\n",
      "Processed 8,800,000 lines: 0 bad lines : 3.85% complete\n",
      "Processed 8,800,000 lines: 0 bad lines : 3.85% complete\n",
      "Processed 8,900,000 lines: 0 bad lines : 3.89% complete\n",
      "Processed 8,900,000 lines: 0 bad lines : 3.89% complete\n",
      "Processed 9,000,000 lines: 0 bad lines : 3.94% complete\n",
      "Processed 9,000,000 lines: 0 bad lines : 3.94% complete\n",
      "Processed 9,100,000 lines: 0 bad lines : 3.98% complete\n",
      "Processed 9,100,000 lines: 0 bad lines : 3.98% complete\n",
      "Processed 9,200,000 lines: 0 bad lines : 4.02% complete\n",
      "Processed 9,200,000 lines: 0 bad lines : 4.02% complete\n",
      "Processed 9,300,000 lines: 0 bad lines : 4.11% complete\n",
      "Processed 9,300,000 lines: 0 bad lines : 4.11% complete\n",
      "Processed 9,400,000 lines: 0 bad lines : 4.15% complete\n",
      "Processed 9,400,000 lines: 0 bad lines : 4.15% complete\n",
      "Processed 9,500,000 lines: 0 bad lines : 4.19% complete\n",
      "Processed 9,500,000 lines: 0 bad lines : 4.19% complete\n",
      "Processed 9,600,000 lines: 0 bad lines : 4.24% complete\n",
      "Processed 9,600,000 lines: 0 bad lines : 4.24% complete\n",
      "Processed 9,700,000 lines: 0 bad lines : 4.28% complete\n",
      "Processed 9,700,000 lines: 0 bad lines : 4.28% complete\n",
      "Processed 9,800,000 lines: 0 bad lines : 4.33% complete\n",
      "Processed 9,800,000 lines: 0 bad lines : 4.33% complete\n",
      "Processed 9,900,000 lines: 0 bad lines : 4.37% complete\n",
      "Processed 9,900,000 lines: 0 bad lines : 4.37% complete\n",
      "Processed 10,000,000 lines: 0 bad lines : 4.41% complete\n",
      "Processed 10,000,000 lines: 0 bad lines : 4.41% complete\n",
      "Processed 10,100,000 lines: 0 bad lines : 4.46% complete\n",
      "Processed 10,100,000 lines: 0 bad lines : 4.46% complete\n",
      "Processed 10,200,000 lines: 0 bad lines : 4.50% complete\n",
      "Processed 10,200,000 lines: 0 bad lines : 4.50% complete\n",
      "Processed 10,300,000 lines: 0 bad lines : 4.55% complete\n",
      "Processed 10,300,000 lines: 0 bad lines : 4.55% complete\n",
      "Processed 10,400,000 lines: 0 bad lines : 4.59% complete\n",
      "Processed 10,400,000 lines: 0 bad lines : 4.59% complete\n",
      "Processed 10,500,000 lines: 0 bad lines : 4.64% complete\n",
      "Processed 10,500,000 lines: 0 bad lines : 4.64% complete\n",
      "Processed 10,600,000 lines: 0 bad lines : 4.68% complete\n",
      "Processed 10,600,000 lines: 0 bad lines : 4.68% complete\n",
      "Processed 10,700,000 lines: 0 bad lines : 4.72% complete\n",
      "Processed 10,700,000 lines: 0 bad lines : 4.72% complete\n",
      "Processed 10,800,000 lines: 0 bad lines : 4.77% complete\n",
      "Processed 10,800,000 lines: 0 bad lines : 4.77% complete\n",
      "Processed 10,900,000 lines: 0 bad lines : 4.81% complete\n",
      "Processed 10,900,000 lines: 0 bad lines : 4.81% complete\n",
      "Processed 11,000,000 lines: 0 bad lines : 4.86% complete\n",
      "Processed 11,000,000 lines: 0 bad lines : 4.86% complete\n",
      "Processed 11,100,000 lines: 0 bad lines : 4.90% complete\n",
      "Processed 11,100,000 lines: 0 bad lines : 4.90% complete\n",
      "Processed 11,200,000 lines: 0 bad lines : 4.95% complete\n",
      "Processed 11,200,000 lines: 0 bad lines : 4.95% complete\n",
      "Processed 11,300,000 lines: 0 bad lines : 4.99% complete\n",
      "Processed 11,300,000 lines: 0 bad lines : 4.99% complete\n",
      "Processed 11,400,000 lines: 0 bad lines : 5.04% complete\n",
      "Processed 11,400,000 lines: 0 bad lines : 5.04% complete\n",
      "Processed 11,500,000 lines: 0 bad lines : 5.08% complete\n",
      "Processed 11,500,000 lines: 0 bad lines : 5.08% complete\n",
      "Processed 11,600,000 lines: 0 bad lines : 5.12% complete\n",
      "Processed 11,600,000 lines: 0 bad lines : 5.12% complete\n",
      "Processed 11,700,000 lines: 0 bad lines : 5.21% complete\n",
      "Processed 11,700,000 lines: 0 bad lines : 5.21% complete\n",
      "Processed 11,800,000 lines: 0 bad lines : 5.26% complete\n",
      "Processed 11,800,000 lines: 0 bad lines : 5.26% complete\n",
      "Processed 11,900,000 lines: 0 bad lines : 5.30% complete\n",
      "Processed 11,900,000 lines: 0 bad lines : 5.30% complete\n",
      "Processed 12,000,000 lines: 0 bad lines : 5.34% complete\n",
      "Processed 12,000,000 lines: 0 bad lines : 5.34% complete\n",
      "Processed 12,100,000 lines: 0 bad lines : 5.39% complete\n",
      "Processed 12,100,000 lines: 0 bad lines : 5.39% complete\n",
      "Processed 12,200,000 lines: 0 bad lines : 5.43% complete\n",
      "Processed 12,200,000 lines: 0 bad lines : 5.43% complete\n",
      "Processed 12,300,000 lines: 0 bad lines : 5.47% complete\n",
      "Processed 12,300,000 lines: 0 bad lines : 5.47% complete\n",
      "Processed 12,400,000 lines: 0 bad lines : 5.52% complete\n",
      "Processed 12,400,000 lines: 0 bad lines : 5.52% complete\n",
      "Processed 12,500,000 lines: 0 bad lines : 5.56% complete\n",
      "Processed 12,500,000 lines: 0 bad lines : 5.56% complete\n",
      "Processed 12,600,000 lines: 0 bad lines : 5.61% complete\n",
      "Processed 12,600,000 lines: 0 bad lines : 5.61% complete\n",
      "Processed 12,700,000 lines: 0 bad lines : 5.65% complete\n",
      "Processed 12,700,000 lines: 0 bad lines : 5.65% complete\n",
      "Processed 12,800,000 lines: 0 bad lines : 5.69% complete\n",
      "Processed 12,800,000 lines: 0 bad lines : 5.69% complete\n",
      "Processed 12,900,000 lines: 0 bad lines : 5.74% complete\n",
      "Processed 12,900,000 lines: 0 bad lines : 5.74% complete\n",
      "Processed 13,000,000 lines: 0 bad lines : 5.78% complete\n",
      "Processed 13,000,000 lines: 0 bad lines : 5.78% complete\n",
      "Processed 13,100,000 lines: 0 bad lines : 5.82% complete\n",
      "Processed 13,100,000 lines: 0 bad lines : 5.82% complete\n",
      "Processed 13,200,000 lines: 0 bad lines : 5.87% complete\n",
      "Processed 13,200,000 lines: 0 bad lines : 5.87% complete\n",
      "Processed 13,300,000 lines: 0 bad lines : 5.91% complete\n",
      "Processed 13,300,000 lines: 0 bad lines : 5.91% complete\n",
      "Processed 13,400,000 lines: 0 bad lines : 5.95% complete\n",
      "Processed 13,400,000 lines: 0 bad lines : 5.95% complete\n",
      "Processed 13,500,000 lines: 0 bad lines : 5.99% complete\n",
      "Processed 13,500,000 lines: 0 bad lines : 5.99% complete\n",
      "Processed 13,600,000 lines: 0 bad lines : 6.04% complete\n",
      "Processed 13,600,000 lines: 0 bad lines : 6.04% complete\n",
      "Processed 13,700,000 lines: 0 bad lines : 6.08% complete\n",
      "Processed 13,700,000 lines: 0 bad lines : 6.08% complete\n",
      "Processed 13,800,000 lines: 0 bad lines : 6.12% complete\n",
      "Processed 13,800,000 lines: 0 bad lines : 6.12% complete\n",
      "Processed 13,900,000 lines: 0 bad lines : 6.16% complete\n",
      "Processed 13,900,000 lines: 0 bad lines : 6.16% complete\n",
      "Processed 14,000,000 lines: 0 bad lines : 6.20% complete\n",
      "Processed 14,000,000 lines: 0 bad lines : 6.20% complete\n",
      "Processed 14,100,000 lines: 0 bad lines : 6.24% complete\n",
      "Processed 14,100,000 lines: 0 bad lines : 6.24% complete\n",
      "Processed 14,200,000 lines: 0 bad lines : 6.29% complete\n",
      "Processed 14,200,000 lines: 0 bad lines : 6.29% complete\n",
      "Processed 14,300,000 lines: 0 bad lines : 6.33% complete\n",
      "Processed 14,300,000 lines: 0 bad lines : 6.33% complete\n",
      "Processed 14,400,000 lines: 0 bad lines : 6.37% complete\n",
      "Processed 14,400,000 lines: 0 bad lines : 6.37% complete\n",
      "Processed 14,500,000 lines: 0 bad lines : 6.45% complete\n",
      "Processed 14,500,000 lines: 0 bad lines : 6.45% complete\n",
      "Processed 14,600,000 lines: 0 bad lines : 6.49% complete\n",
      "Processed 14,600,000 lines: 0 bad lines : 6.49% complete\n",
      "Processed 14,700,000 lines: 0 bad lines : 6.54% complete\n",
      "Processed 14,700,000 lines: 0 bad lines : 6.54% complete\n",
      "Processed 14,800,000 lines: 0 bad lines : 6.58% complete\n",
      "Processed 14,800,000 lines: 0 bad lines : 6.58% complete\n",
      "Processed 14,900,000 lines: 0 bad lines : 6.62% complete\n",
      "Processed 14,900,000 lines: 0 bad lines : 6.62% complete\n",
      "Processed 15,000,000 lines: 0 bad lines : 6.66% complete\n",
      "Processed 15,000,000 lines: 0 bad lines : 6.66% complete\n",
      "Processed 15,100,000 lines: 0 bad lines : 6.70% complete\n",
      "Processed 15,100,000 lines: 0 bad lines : 6.70% complete\n",
      "Processed 15,200,000 lines: 0 bad lines : 6.75% complete\n",
      "Processed 15,200,000 lines: 0 bad lines : 6.75% complete\n",
      "Processed 15,300,000 lines: 0 bad lines : 6.79% complete\n",
      "Processed 15,300,000 lines: 0 bad lines : 6.79% complete\n",
      "Processed 15,400,000 lines: 0 bad lines : 6.83% complete\n",
      "Processed 15,400,000 lines: 0 bad lines : 6.83% complete\n",
      "Processed 15,500,000 lines: 0 bad lines : 6.87% complete\n",
      "Processed 15,500,000 lines: 0 bad lines : 6.87% complete\n",
      "Processed 15,600,000 lines: 0 bad lines : 6.92% complete\n",
      "Processed 15,600,000 lines: 0 bad lines : 6.92% complete\n",
      "Processed 15,700,000 lines: 0 bad lines : 6.96% complete\n",
      "Processed 15,700,000 lines: 0 bad lines : 6.96% complete\n",
      "Processed 15,800,000 lines: 0 bad lines : 7.00% complete\n",
      "Processed 15,800,000 lines: 0 bad lines : 7.00% complete\n",
      "Processed 15,900,000 lines: 0 bad lines : 7.04% complete\n",
      "Processed 15,900,000 lines: 0 bad lines : 7.04% complete\n",
      "Processed 16,000,000 lines: 0 bad lines : 7.09% complete\n",
      "Processed 16,000,000 lines: 0 bad lines : 7.09% complete\n",
      "Processed 16,100,000 lines: 0 bad lines : 7.13% complete\n",
      "Processed 16,100,000 lines: 0 bad lines : 7.13% complete\n",
      "Processed 16,200,000 lines: 0 bad lines : 7.17% complete\n",
      "Processed 16,200,000 lines: 0 bad lines : 7.17% complete\n",
      "Processed 16,300,000 lines: 0 bad lines : 7.21% complete\n",
      "Processed 16,300,000 lines: 0 bad lines : 7.21% complete\n",
      "Processed 16,400,000 lines: 0 bad lines : 7.26% complete\n",
      "Processed 16,400,000 lines: 0 bad lines : 7.26% complete\n",
      "Processed 16,500,000 lines: 0 bad lines : 7.30% complete\n",
      "Processed 16,500,000 lines: 0 bad lines : 7.30% complete\n",
      "Processed 16,600,000 lines: 0 bad lines : 7.34% complete\n",
      "Processed 16,600,000 lines: 0 bad lines : 7.34% complete\n",
      "Processed 16,700,000 lines: 0 bad lines : 7.39% complete\n",
      "Processed 16,700,000 lines: 0 bad lines : 7.39% complete\n",
      "Processed 16,800,000 lines: 0 bad lines : 7.43% complete\n",
      "Processed 16,800,000 lines: 0 bad lines : 7.43% complete\n",
      "Processed 16,900,000 lines: 0 bad lines : 7.47% complete\n",
      "Processed 16,900,000 lines: 0 bad lines : 7.47% complete\n",
      "Processed 17,000,000 lines: 0 bad lines : 7.52% complete\n",
      "Processed 17,000,000 lines: 0 bad lines : 7.52% complete\n",
      "Processed 17,100,000 lines: 0 bad lines : 7.56% complete\n",
      "Processed 17,100,000 lines: 0 bad lines : 7.56% complete\n",
      "Processed 17,200,000 lines: 0 bad lines : 7.60% complete\n",
      "Processed 17,200,000 lines: 0 bad lines : 7.60% complete\n",
      "Processed 17,300,000 lines: 0 bad lines : 7.69% complete\n",
      "Processed 17,300,000 lines: 0 bad lines : 7.69% complete\n",
      "Processed 17,400,000 lines: 0 bad lines : 7.74% complete\n",
      "Processed 17,400,000 lines: 0 bad lines : 7.74% complete\n",
      "Processed 17,500,000 lines: 0 bad lines : 7.78% complete\n",
      "Processed 17,500,000 lines: 0 bad lines : 7.78% complete\n",
      "Processed 17,600,000 lines: 0 bad lines : 7.83% complete\n",
      "Processed 17,600,000 lines: 0 bad lines : 7.83% complete\n",
      "Processed 17,700,000 lines: 0 bad lines : 7.87% complete\n",
      "Processed 17,700,000 lines: 0 bad lines : 7.87% complete\n",
      "Processed 17,800,000 lines: 0 bad lines : 7.92% complete\n",
      "Processed 17,800,000 lines: 0 bad lines : 7.92% complete\n",
      "Processed 17,900,000 lines: 0 bad lines : 7.96% complete\n",
      "Processed 17,900,000 lines: 0 bad lines : 7.96% complete\n",
      "Processed 18,000,000 lines: 0 bad lines : 8.01% complete\n",
      "Processed 18,000,000 lines: 0 bad lines : 8.01% complete\n",
      "Processed 18,100,000 lines: 0 bad lines : 8.05% complete\n",
      "Processed 18,100,000 lines: 0 bad lines : 8.05% complete\n",
      "Processed 18,200,000 lines: 0 bad lines : 8.09% complete\n",
      "Processed 18,200,000 lines: 0 bad lines : 8.09% complete\n",
      "Processed 18,300,000 lines: 0 bad lines : 8.14% complete\n",
      "Processed 18,300,000 lines: 0 bad lines : 8.14% complete\n",
      "Processed 18,400,000 lines: 0 bad lines : 8.18% complete\n",
      "Processed 18,400,000 lines: 0 bad lines : 8.18% complete\n",
      "Processed 18,500,000 lines: 0 bad lines : 8.23% complete\n",
      "Processed 18,500,000 lines: 0 bad lines : 8.23% complete\n",
      "Processed 18,600,000 lines: 0 bad lines : 8.27% complete\n",
      "Processed 18,600,000 lines: 0 bad lines : 8.27% complete\n",
      "Processed 18,700,000 lines: 0 bad lines : 8.32% complete\n",
      "Processed 18,700,000 lines: 0 bad lines : 8.32% complete\n",
      "Processed 18,800,000 lines: 0 bad lines : 8.36% complete\n",
      "Processed 18,800,000 lines: 0 bad lines : 8.36% complete\n",
      "Processed 18,900,000 lines: 0 bad lines : 8.40% complete\n",
      "Processed 18,900,000 lines: 0 bad lines : 8.40% complete\n",
      "Processed 19,000,000 lines: 0 bad lines : 8.45% complete\n",
      "Processed 19,000,000 lines: 0 bad lines : 8.45% complete\n",
      "Processed 19,100,000 lines: 0 bad lines : 8.49% complete\n",
      "Processed 19,100,000 lines: 0 bad lines : 8.49% complete\n",
      "Processed 19,200,000 lines: 0 bad lines : 8.53% complete\n",
      "Processed 19,200,000 lines: 0 bad lines : 8.53% complete\n",
      "Processed 19,300,000 lines: 0 bad lines : 8.58% complete\n",
      "Processed 19,300,000 lines: 0 bad lines : 8.58% complete\n",
      "Processed 19,400,000 lines: 0 bad lines : 8.62% complete\n",
      "Processed 19,400,000 lines: 0 bad lines : 8.62% complete\n",
      "Processed 19,500,000 lines: 0 bad lines : 8.67% complete\n",
      "Processed 19,500,000 lines: 0 bad lines : 8.67% complete\n",
      "Processed 19,600,000 lines: 0 bad lines : 8.71% complete\n",
      "Processed 19,600,000 lines: 0 bad lines : 8.71% complete\n",
      "Processed 19,700,000 lines: 0 bad lines : 8.80% complete\n",
      "Processed 19,700,000 lines: 0 bad lines : 8.80% complete\n",
      "Processed 19,800,000 lines: 0 bad lines : 8.84% complete\n",
      "Processed 19,800,000 lines: 0 bad lines : 8.84% complete\n",
      "Processed 19,900,000 lines: 0 bad lines : 8.88% complete\n",
      "Processed 19,900,000 lines: 0 bad lines : 8.88% complete\n",
      "Processed 20,000,000 lines: 0 bad lines : 8.93% complete\n",
      "Processed 20,000,000 lines: 0 bad lines : 8.93% complete\n",
      "Processed 20,100,000 lines: 0 bad lines : 8.97% complete\n",
      "Processed 20,100,000 lines: 0 bad lines : 8.97% complete\n",
      "Processed 20,200,000 lines: 0 bad lines : 9.01% complete\n",
      "Processed 20,200,000 lines: 0 bad lines : 9.01% complete\n",
      "Processed 20,300,000 lines: 0 bad lines : 9.06% complete\n",
      "Processed 20,300,000 lines: 0 bad lines : 9.06% complete\n",
      "Processed 20,400,000 lines: 0 bad lines : 9.10% complete\n",
      "Processed 20,400,000 lines: 0 bad lines : 9.10% complete\n",
      "Processed 20,500,000 lines: 0 bad lines : 9.14% complete\n",
      "Processed 20,500,000 lines: 0 bad lines : 9.14% complete\n",
      "Processed 20,600,000 lines: 0 bad lines : 9.19% complete\n",
      "Processed 20,600,000 lines: 0 bad lines : 9.19% complete\n",
      "Processed 20,700,000 lines: 0 bad lines : 9.23% complete\n",
      "Processed 20,700,000 lines: 0 bad lines : 9.23% complete\n",
      "Processed 20,800,000 lines: 0 bad lines : 9.27% complete\n",
      "Processed 20,800,000 lines: 0 bad lines : 9.27% complete\n",
      "Processed 20,900,000 lines: 0 bad lines : 9.32% complete\n",
      "Processed 20,900,000 lines: 0 bad lines : 9.32% complete\n",
      "Processed 21,000,000 lines: 0 bad lines : 9.36% complete\n",
      "Processed 21,000,000 lines: 0 bad lines : 9.36% complete\n",
      "Processed 21,100,000 lines: 0 bad lines : 9.40% complete\n",
      "Processed 21,100,000 lines: 0 bad lines : 9.40% complete\n",
      "Processed 21,200,000 lines: 0 bad lines : 9.44% complete\n",
      "Processed 21,200,000 lines: 0 bad lines : 9.44% complete\n",
      "Processed 21,300,000 lines: 0 bad lines : 9.48% complete\n",
      "Processed 21,300,000 lines: 0 bad lines : 9.48% complete\n",
      "Processed 21,400,000 lines: 0 bad lines : 9.53% complete\n",
      "Processed 21,400,000 lines: 0 bad lines : 9.53% complete\n",
      "Processed 21,500,000 lines: 0 bad lines : 9.57% complete\n",
      "Processed 21,500,000 lines: 0 bad lines : 9.57% complete\n",
      "Processed 21,600,000 lines: 0 bad lines : 9.61% complete\n",
      "Processed 21,600,000 lines: 0 bad lines : 9.61% complete\n",
      "Processed 21,700,000 lines: 0 bad lines : 9.65% complete\n",
      "Processed 21,700,000 lines: 0 bad lines : 9.65% complete\n",
      "Processed 21,800,000 lines: 0 bad lines : 9.69% complete\n",
      "Processed 21,800,000 lines: 0 bad lines : 9.69% complete\n",
      "Processed 21,900,000 lines: 0 bad lines : 9.74% complete\n",
      "Processed 21,900,000 lines: 0 bad lines : 9.74% complete\n",
      "Processed 22,000,000 lines: 0 bad lines : 9.78% complete\n",
      "Processed 22,000,000 lines: 0 bad lines : 9.78% complete\n",
      "Processed 22,100,000 lines: 0 bad lines : 9.82% complete\n",
      "Processed 22,100,000 lines: 0 bad lines : 9.82% complete\n",
      "Processed 22,200,000 lines: 0 bad lines : 9.86% complete\n",
      "Processed 22,200,000 lines: 0 bad lines : 9.86% complete\n",
      "Processed 22,300,000 lines: 0 bad lines : 9.90% complete\n",
      "Processed 22,300,000 lines: 0 bad lines : 9.90% complete\n",
      "Processed 22,400,000 lines: 0 bad lines : 9.95% complete\n",
      "Processed 22,400,000 lines: 0 bad lines : 9.95% complete\n",
      "Processed 22,500,000 lines: 0 bad lines : 9.99% complete\n",
      "Processed 22,500,000 lines: 0 bad lines : 9.99% complete\n",
      "Processed 22,600,000 lines: 0 bad lines : 10.07% complete\n",
      "Processed 22,600,000 lines: 0 bad lines : 10.07% complete\n",
      "Processed 22,700,000 lines: 0 bad lines : 10.12% complete\n",
      "Processed 22,700,000 lines: 0 bad lines : 10.12% complete\n",
      "Processed 22,800,000 lines: 0 bad lines : 10.16% complete\n",
      "Processed 22,800,000 lines: 0 bad lines : 10.16% complete\n",
      "Processed 22,900,000 lines: 0 bad lines : 10.20% complete\n",
      "Processed 22,900,000 lines: 0 bad lines : 10.20% complete\n",
      "Processed 23,000,000 lines: 0 bad lines : 10.25% complete\n",
      "Processed 23,000,000 lines: 0 bad lines : 10.25% complete\n",
      "Processed 23,100,000 lines: 0 bad lines : 10.29% complete\n",
      "Processed 23,100,000 lines: 0 bad lines : 10.29% complete\n",
      "Processed 23,200,000 lines: 0 bad lines : 10.33% complete\n",
      "Processed 23,200,000 lines: 0 bad lines : 10.33% complete\n",
      "Processed 23,300,000 lines: 0 bad lines : 10.37% complete\n",
      "Processed 23,300,000 lines: 0 bad lines : 10.37% complete\n",
      "Processed 23,400,000 lines: 0 bad lines : 10.42% complete\n",
      "Processed 23,400,000 lines: 0 bad lines : 10.42% complete\n",
      "Processed 23,500,000 lines: 0 bad lines : 10.46% complete\n",
      "Processed 23,500,000 lines: 0 bad lines : 10.46% complete\n",
      "Processed 23,600,000 lines: 0 bad lines : 10.50% complete\n",
      "Processed 23,600,000 lines: 0 bad lines : 10.50% complete\n",
      "Processed 23,700,000 lines: 0 bad lines : 10.55% complete\n",
      "Processed 23,700,000 lines: 0 bad lines : 10.55% complete\n",
      "Processed 23,800,000 lines: 0 bad lines : 10.59% complete\n",
      "Processed 23,800,000 lines: 0 bad lines : 10.59% complete\n",
      "Processed 23,900,000 lines: 0 bad lines : 10.63% complete\n",
      "Processed 23,900,000 lines: 0 bad lines : 10.63% complete\n",
      "Processed 24,000,000 lines: 0 bad lines : 10.68% complete\n",
      "Processed 24,000,000 lines: 0 bad lines : 10.68% complete\n",
      "Processed 24,100,000 lines: 0 bad lines : 10.72% complete\n",
      "Processed 24,100,000 lines: 0 bad lines : 10.72% complete\n",
      "Processed 24,200,000 lines: 0 bad lines : 10.76% complete\n",
      "Processed 24,200,000 lines: 0 bad lines : 10.76% complete\n",
      "Processed 24,300,000 lines: 0 bad lines : 10.81% complete\n",
      "Processed 24,300,000 lines: 0 bad lines : 10.81% complete\n",
      "Processed 24,400,000 lines: 0 bad lines : 10.85% complete\n",
      "Processed 24,400,000 lines: 0 bad lines : 10.85% complete\n",
      "Processed 24,500,000 lines: 0 bad lines : 10.89% complete\n",
      "Processed 24,500,000 lines: 0 bad lines : 10.89% complete\n",
      "Processed 24,600,000 lines: 0 bad lines : 10.94% complete\n",
      "Processed 24,600,000 lines: 0 bad lines : 10.94% complete\n",
      "Processed 24,700,000 lines: 0 bad lines : 10.98% complete\n",
      "Processed 24,700,000 lines: 0 bad lines : 10.98% complete\n",
      "Processed 24,800,000 lines: 0 bad lines : 11.03% complete\n",
      "Processed 24,800,000 lines: 0 bad lines : 11.03% complete\n",
      "Processed 24,900,000 lines: 0 bad lines : 11.07% complete\n",
      "Processed 24,900,000 lines: 0 bad lines : 11.07% complete\n",
      "Processed 25,000,000 lines: 0 bad lines : 11.12% complete\n",
      "Processed 25,000,000 lines: 0 bad lines : 11.12% complete\n",
      "Processed 25,100,000 lines: 0 bad lines : 11.16% complete\n",
      "Processed 25,100,000 lines: 0 bad lines : 11.16% complete\n",
      "Processed 25,200,000 lines: 0 bad lines : 11.20% complete\n",
      "Processed 25,200,000 lines: 0 bad lines : 11.20% complete\n",
      "Processed 25,300,000 lines: 0 bad lines : 11.25% complete\n",
      "Processed 25,300,000 lines: 0 bad lines : 11.25% complete\n",
      "Processed 25,400,000 lines: 0 bad lines : 11.34% complete\n",
      "Processed 25,400,000 lines: 0 bad lines : 11.34% complete\n",
      "Processed 25,500,000 lines: 0 bad lines : 11.38% complete\n",
      "Processed 25,500,000 lines: 0 bad lines : 11.38% complete\n",
      "Processed 25,600,000 lines: 0 bad lines : 11.43% complete\n",
      "Processed 25,600,000 lines: 0 bad lines : 11.43% complete\n",
      "Processed 25,700,000 lines: 0 bad lines : 11.47% complete\n",
      "Processed 25,700,000 lines: 0 bad lines : 11.47% complete\n",
      "Processed 25,800,000 lines: 0 bad lines : 11.51% complete\n",
      "Processed 25,800,000 lines: 0 bad lines : 11.51% complete\n",
      "Processed 25,900,000 lines: 0 bad lines : 11.56% complete\n",
      "Processed 25,900,000 lines: 0 bad lines : 11.56% complete\n",
      "Processed 26,000,000 lines: 0 bad lines : 11.60% complete\n",
      "Processed 26,000,000 lines: 0 bad lines : 11.60% complete\n",
      "Processed 26,100,000 lines: 0 bad lines : 11.65% complete\n",
      "Processed 26,100,000 lines: 0 bad lines : 11.65% complete\n",
      "Processed 26,200,000 lines: 0 bad lines : 11.69% complete\n",
      "Processed 26,200,000 lines: 0 bad lines : 11.69% complete\n",
      "Processed 26,300,000 lines: 0 bad lines : 11.73% complete\n",
      "Processed 26,300,000 lines: 0 bad lines : 11.73% complete\n",
      "Processed 26,400,000 lines: 0 bad lines : 11.78% complete\n",
      "Processed 26,400,000 lines: 0 bad lines : 11.78% complete\n",
      "Processed 26,500,000 lines: 0 bad lines : 11.82% complete\n",
      "Processed 26,500,000 lines: 0 bad lines : 11.82% complete\n",
      "Processed 26,600,000 lines: 0 bad lines : 11.87% complete\n",
      "Processed 26,600,000 lines: 0 bad lines : 11.87% complete\n",
      "Processed 26,700,000 lines: 0 bad lines : 11.91% complete\n",
      "Processed 26,700,000 lines: 0 bad lines : 11.91% complete\n",
      "Processed 26,800,000 lines: 0 bad lines : 11.95% complete\n",
      "Processed 26,800,000 lines: 0 bad lines : 11.95% complete\n",
      "Processed 26,900,000 lines: 0 bad lines : 12.00% complete\n",
      "Processed 26,900,000 lines: 0 bad lines : 12.00% complete\n",
      "Processed 27,000,000 lines: 0 bad lines : 12.04% complete\n",
      "Processed 27,000,000 lines: 0 bad lines : 12.04% complete\n",
      "Processed 27,100,000 lines: 0 bad lines : 12.08% complete\n",
      "Processed 27,100,000 lines: 0 bad lines : 12.08% complete\n",
      "Processed 27,200,000 lines: 0 bad lines : 12.13% complete\n",
      "Processed 27,200,000 lines: 0 bad lines : 12.13% complete\n",
      "Processed 27,300,000 lines: 0 bad lines : 12.17% complete\n",
      "Processed 27,300,000 lines: 0 bad lines : 12.17% complete\n",
      "Processed 27,400,000 lines: 0 bad lines : 12.21% complete\n",
      "Processed 27,400,000 lines: 0 bad lines : 12.21% complete\n",
      "Processed 27,500,000 lines: 0 bad lines : 12.26% complete\n",
      "Processed 27,500,000 lines: 0 bad lines : 12.26% complete\n",
      "Processed 27,600,000 lines: 0 bad lines : 12.30% complete\n",
      "Processed 27,600,000 lines: 0 bad lines : 12.30% complete\n",
      "Processed 27,700,000 lines: 0 bad lines : 12.34% complete\n",
      "Processed 27,700,000 lines: 0 bad lines : 12.34% complete\n",
      "Processed 27,800,000 lines: 0 bad lines : 12.39% complete\n",
      "Processed 27,800,000 lines: 0 bad lines : 12.39% complete\n",
      "Processed 27,900,000 lines: 0 bad lines : 12.43% complete\n",
      "Processed 27,900,000 lines: 0 bad lines : 12.43% complete\n",
      "Processed 28,000,000 lines: 0 bad lines : 12.47% complete\n",
      "Processed 28,000,000 lines: 0 bad lines : 12.47% complete\n",
      "Processed 28,100,000 lines: 0 bad lines : 12.56% complete\n",
      "Processed 28,100,000 lines: 0 bad lines : 12.56% complete\n",
      "Processed 28,200,000 lines: 0 bad lines : 12.60% complete\n",
      "Processed 28,200,000 lines: 0 bad lines : 12.60% complete\n",
      "Processed 28,300,000 lines: 0 bad lines : 12.64% complete\n",
      "Processed 28,300,000 lines: 0 bad lines : 12.64% complete\n",
      "Processed 28,400,000 lines: 0 bad lines : 12.68% complete\n",
      "Processed 28,400,000 lines: 0 bad lines : 12.68% complete\n",
      "Processed 28,500,000 lines: 0 bad lines : 12.73% complete\n",
      "Processed 28,500,000 lines: 0 bad lines : 12.73% complete\n",
      "Processed 28,600,000 lines: 0 bad lines : 12.77% complete\n",
      "Processed 28,600,000 lines: 0 bad lines : 12.77% complete\n",
      "Processed 28,700,000 lines: 0 bad lines : 12.81% complete\n",
      "Processed 28,700,000 lines: 0 bad lines : 12.81% complete\n",
      "Processed 28,800,000 lines: 0 bad lines : 12.85% complete\n",
      "Processed 28,800,000 lines: 0 bad lines : 12.85% complete\n",
      "Processed 28,900,000 lines: 0 bad lines : 12.89% complete\n",
      "Processed 28,900,000 lines: 0 bad lines : 12.89% complete\n",
      "Processed 29,000,000 lines: 0 bad lines : 12.93% complete\n",
      "Processed 29,000,000 lines: 0 bad lines : 12.93% complete\n",
      "Processed 29,100,000 lines: 0 bad lines : 12.98% complete\n",
      "Processed 29,100,000 lines: 0 bad lines : 12.98% complete\n",
      "Processed 29,200,000 lines: 0 bad lines : 13.02% complete\n",
      "Processed 29,200,000 lines: 0 bad lines : 13.02% complete\n",
      "Processed 29,300,000 lines: 0 bad lines : 13.06% complete\n",
      "Processed 29,300,000 lines: 0 bad lines : 13.06% complete\n",
      "Processed 29,400,000 lines: 0 bad lines : 13.10% complete\n",
      "Processed 29,400,000 lines: 0 bad lines : 13.10% complete\n",
      "Processed 29,500,000 lines: 0 bad lines : 13.14% complete\n",
      "Processed 29,500,000 lines: 0 bad lines : 13.14% complete\n",
      "Processed 29,600,000 lines: 0 bad lines : 13.18% complete\n",
      "Processed 29,600,000 lines: 0 bad lines : 13.18% complete\n",
      "Processed 29,700,000 lines: 0 bad lines : 13.22% complete\n",
      "Processed 29,700,000 lines: 0 bad lines : 13.22% complete\n",
      "Processed 29,800,000 lines: 0 bad lines : 13.27% complete\n",
      "Processed 29,800,000 lines: 0 bad lines : 13.27% complete\n",
      "Processed 29,900,000 lines: 0 bad lines : 13.31% complete\n",
      "Processed 29,900,000 lines: 0 bad lines : 13.31% complete\n",
      "Processed 30,000,000 lines: 0 bad lines : 13.35% complete\n",
      "Processed 30,000,000 lines: 0 bad lines : 13.35% complete\n",
      "Processed 30,100,000 lines: 0 bad lines : 13.39% complete\n",
      "Processed 30,100,000 lines: 0 bad lines : 13.39% complete\n",
      "Processed 30,200,000 lines: 0 bad lines : 13.43% complete\n",
      "Processed 30,200,000 lines: 0 bad lines : 13.43% complete\n",
      "Processed 30,300,000 lines: 0 bad lines : 13.48% complete\n",
      "Processed 30,300,000 lines: 0 bad lines : 13.48% complete\n",
      "Processed 30,400,000 lines: 0 bad lines : 13.52% complete\n",
      "Processed 30,400,000 lines: 0 bad lines : 13.52% complete\n",
      "Processed 30,500,000 lines: 0 bad lines : 13.56% complete\n",
      "Processed 30,500,000 lines: 0 bad lines : 13.56% complete\n",
      "Processed 30,600,000 lines: 0 bad lines : 13.61% complete\n",
      "Processed 30,600,000 lines: 0 bad lines : 13.61% complete\n",
      "Processed 30,700,000 lines: 0 bad lines : 13.65% complete\n",
      "Processed 30,700,000 lines: 0 bad lines : 13.65% complete\n",
      "Processed 30,800,000 lines: 0 bad lines : 13.69% complete\n",
      "Processed 30,800,000 lines: 0 bad lines : 13.69% complete\n",
      "Processed 30,900,000 lines: 0 bad lines : 13.73% complete\n",
      "Processed 30,900,000 lines: 0 bad lines : 13.73% complete\n",
      "Processed 31,000,000 lines: 0 bad lines : 13.78% complete\n",
      "Processed 31,000,000 lines: 0 bad lines : 13.78% complete\n",
      "Processed 31,100,000 lines: 0 bad lines : 13.82% complete\n",
      "Processed 31,100,000 lines: 0 bad lines : 13.82% complete\n",
      "Processed 31,200,000 lines: 0 bad lines : 13.86% complete\n",
      "Processed 31,200,000 lines: 0 bad lines : 13.86% complete\n",
      "Processed 31,300,000 lines: 0 bad lines : 13.90% complete\n",
      "Processed 31,300,000 lines: 0 bad lines : 13.90% complete\n",
      "Processed 31,400,000 lines: 0 bad lines : 13.99% complete\n",
      "Processed 31,400,000 lines: 0 bad lines : 13.99% complete\n",
      "Processed 31,500,000 lines: 0 bad lines : 14.03% complete\n",
      "Processed 31,500,000 lines: 0 bad lines : 14.03% complete\n",
      "Processed 31,600,000 lines: 0 bad lines : 14.08% complete\n",
      "Processed 31,600,000 lines: 0 bad lines : 14.08% complete\n",
      "Processed 31,700,000 lines: 0 bad lines : 14.12% complete\n",
      "Processed 31,700,000 lines: 0 bad lines : 14.12% complete\n",
      "Processed 31,800,000 lines: 0 bad lines : 14.16% complete\n",
      "Processed 31,800,000 lines: 0 bad lines : 14.16% complete\n",
      "Processed 31,900,000 lines: 0 bad lines : 14.18% complete\n",
      "Processed 31,900,000 lines: 0 bad lines : 14.18% complete\n",
      "Processed 32,000,000 lines: 0 bad lines : 14.21% complete\n",
      "Processed 32,000,000 lines: 0 bad lines : 14.21% complete\n",
      "Processed 32,100,000 lines: 0 bad lines : 14.25% complete\n",
      "Processed 32,100,000 lines: 0 bad lines : 14.25% complete\n",
      "Processed 32,200,000 lines: 0 bad lines : 14.29% complete\n",
      "Processed 32,200,000 lines: 0 bad lines : 14.29% complete\n",
      "Processed 32,300,000 lines: 0 bad lines : 14.33% complete\n",
      "Processed 32,300,000 lines: 0 bad lines : 14.33% complete\n",
      "Processed 32,400,000 lines: 0 bad lines : 14.38% complete\n",
      "Processed 32,400,000 lines: 0 bad lines : 14.38% complete\n",
      "Processed 32,500,000 lines: 0 bad lines : 14.42% complete\n",
      "Processed 32,500,000 lines: 0 bad lines : 14.42% complete\n",
      "Processed 32,600,000 lines: 0 bad lines : 14.47% complete\n",
      "Processed 32,600,000 lines: 0 bad lines : 14.47% complete\n",
      "Processed 32,700,000 lines: 0 bad lines : 14.51% complete\n",
      "Processed 32,700,000 lines: 0 bad lines : 14.51% complete\n",
      "Processed 32,800,000 lines: 0 bad lines : 14.56% complete\n",
      "Processed 32,800,000 lines: 0 bad lines : 14.56% complete\n",
      "Processed 32,900,000 lines: 0 bad lines : 14.60% complete\n",
      "Processed 32,900,000 lines: 0 bad lines : 14.60% complete\n",
      "Processed 33,000,000 lines: 0 bad lines : 14.64% complete\n",
      "Processed 33,000,000 lines: 0 bad lines : 14.64% complete\n",
      "Processed 33,100,000 lines: 0 bad lines : 14.69% complete\n",
      "Processed 33,100,000 lines: 0 bad lines : 14.69% complete\n",
      "Processed 33,200,000 lines: 0 bad lines : 14.73% complete\n",
      "Processed 33,200,000 lines: 0 bad lines : 14.73% complete\n",
      "Processed 33,300,000 lines: 0 bad lines : 14.78% complete\n",
      "Processed 33,300,000 lines: 0 bad lines : 14.78% complete\n",
      "Processed 33,400,000 lines: 0 bad lines : 14.82% complete\n",
      "Processed 33,400,000 lines: 0 bad lines : 14.82% complete\n",
      "Processed 33,500,000 lines: 0 bad lines : 14.86% complete\n",
      "Processed 33,500,000 lines: 0 bad lines : 14.86% complete\n",
      "Processed 33,600,000 lines: 0 bad lines : 14.91% complete\n",
      "Processed 33,600,000 lines: 0 bad lines : 14.91% complete\n",
      "Processed 33,700,000 lines: 0 bad lines : 14.95% complete\n",
      "Processed 33,700,000 lines: 0 bad lines : 14.95% complete\n",
      "Processed 33,800,000 lines: 0 bad lines : 15.00% complete\n",
      "Processed 33,800,000 lines: 0 bad lines : 15.00% complete\n",
      "Processed 33,900,000 lines: 0 bad lines : 15.04% complete\n",
      "Processed 33,900,000 lines: 0 bad lines : 15.04% complete\n",
      "Processed 34,000,000 lines: 0 bad lines : 15.08% complete\n",
      "Processed 34,000,000 lines: 0 bad lines : 15.08% complete\n",
      "Processed 34,100,000 lines: 0 bad lines : 15.13% complete\n",
      "Processed 34,100,000 lines: 0 bad lines : 15.13% complete\n",
      "Processed 34,200,000 lines: 0 bad lines : 15.17% complete\n",
      "Processed 34,200,000 lines: 0 bad lines : 15.17% complete\n",
      "Processed 34,300,000 lines: 0 bad lines : 15.21% complete\n",
      "Processed 34,300,000 lines: 0 bad lines : 15.21% complete\n",
      "Processed 34,400,000 lines: 0 bad lines : 15.26% complete\n",
      "Processed 34,400,000 lines: 0 bad lines : 15.26% complete\n",
      "Processed 34,500,000 lines: 0 bad lines : 15.34% complete\n",
      "Processed 34,500,000 lines: 0 bad lines : 15.34% complete\n",
      "Processed 34,600,000 lines: 0 bad lines : 15.39% complete\n",
      "Processed 34,600,000 lines: 0 bad lines : 15.39% complete\n",
      "Processed 34,700,000 lines: 0 bad lines : 15.43% complete\n",
      "Processed 34,700,000 lines: 0 bad lines : 15.43% complete\n",
      "Processed 34,800,000 lines: 0 bad lines : 15.47% complete\n",
      "Processed 34,800,000 lines: 0 bad lines : 15.47% complete\n",
      "Processed 34,900,000 lines: 0 bad lines : 15.52% complete\n",
      "Processed 34,900,000 lines: 0 bad lines : 15.52% complete\n",
      "Processed 35,000,000 lines: 0 bad lines : 15.56% complete\n",
      "Processed 35,000,000 lines: 0 bad lines : 15.56% complete\n",
      "Processed 35,100,000 lines: 0 bad lines : 15.60% complete\n",
      "Processed 35,100,000 lines: 0 bad lines : 15.60% complete\n",
      "Processed 35,200,000 lines: 0 bad lines : 15.65% complete\n",
      "Processed 35,200,000 lines: 0 bad lines : 15.65% complete\n",
      "Processed 35,300,000 lines: 0 bad lines : 15.69% complete\n",
      "Processed 35,300,000 lines: 0 bad lines : 15.69% complete\n",
      "Processed 35,400,000 lines: 0 bad lines : 15.73% complete\n",
      "Processed 35,400,000 lines: 0 bad lines : 15.73% complete\n",
      "Processed 35,500,000 lines: 0 bad lines : 15.77% complete\n",
      "Processed 35,500,000 lines: 0 bad lines : 15.77% complete\n",
      "Processed 35,600,000 lines: 0 bad lines : 15.82% complete\n",
      "Processed 35,600,000 lines: 0 bad lines : 15.82% complete\n",
      "Processed 35,700,000 lines: 0 bad lines : 15.86% complete\n",
      "Processed 35,700,000 lines: 0 bad lines : 15.86% complete\n",
      "Processed 35,800,000 lines: 0 bad lines : 15.90% complete\n",
      "Processed 35,800,000 lines: 0 bad lines : 15.90% complete\n",
      "Processed 35,900,000 lines: 0 bad lines : 15.95% complete\n",
      "Processed 35,900,000 lines: 0 bad lines : 15.95% complete\n"
     ]
    }
   ],
   "source": [
    "\n",
    "log = logging.getLogger(\"bot\")\n",
    "log.setLevel(logging.DEBUG)\n",
    "log.addHandler(logging.StreamHandler())\n",
    "\n",
    "def read_and_decode(reader, chunk_size, max_window_size, previous_chunk=None, bytes_read=0):\n",
    "    chunk = reader.read(chunk_size)\n",
    "    bytes_read += chunk_size\n",
    "    if previous_chunk is not None:\n",
    "        chunk = previous_chunk + chunk\n",
    "    try:\n",
    "        return chunk.decode()\n",
    "    except UnicodeDecodeError:\n",
    "        if bytes_read > max_window_size:\n",
    "            raise UnicodeError(f\"Unable to decode frame after reading {bytes_read:,} bytes\")\n",
    "        return read_and_decode(reader, chunk_size, max_window_size, chunk, bytes_read)\n",
    "\n",
    "def read_lines_zst(file_name):\n",
    "    with open(file_name, 'rb') as file_handle:\n",
    "        buffer = ''\n",
    "        reader = zstandard.ZstdDecompressor(max_window_size=2**31).stream_reader(file_handle)\n",
    "        while True:\n",
    "            chunk = read_and_decode(reader, 2**27, (2**29) * 2)\n",
    "            if not chunk:\n",
    "                break\n",
    "            lines = (buffer + chunk).split(\"\\n\")\n",
    "            for line in lines[:-1]:\n",
    "                yield line, file_handle.tell()\n",
    "            buffer = lines[-1]\n",
    "        reader.close()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    zst_file_name = 'RC_2021-08' \n",
    "    input_file_path = '../../comments/' +zst_file_name+ '.zst'\n",
    "    output_file_path = '../comments/csv/'+zst_file_name+'.csv'\n",
    "    subreddit_to_filter = 'austin'  # The subreddit you want to filter by\n",
    "    subreddits = ['NOLA', 'acadiana', 'centrallouisiana', 'houma', 'louisiana', \n",
    "                  'batonrouge', 'neworleans', 'Thibodaux', 'newyork', 'Adirondacks', \n",
    "                  'bronx', 'Brooklyn', 'Bushwick', 'hudsonvalley', 'OrangeCountyNY', \n",
    "                  'Queens', 'Westchester', 'Buffalo', 'ForestHills', 'longisland', \n",
    "                  'nyc', 'parkslope', 'williamsburg', 'newjersey', 'Hoboken', \n",
    "                  'jerseycity', 'Newark', 'SouthJersey', 'asburypark', 'NewBrunswickNJ', \n",
    "                  'texas', 'Austin', 'CorpusChristi', 'galveston', 'pflugerville', \n",
    "                  'sanantonio', 'corpus', 'Denton', 'houston', 'TriCitiesWA', 'Washington', 'oregon', 'Seattle', 'SeattleWA', 'Portland']\n",
    "\n",
    "    fields = ['id', 'author', 'body', 'link_id', 'parent_id', 'subreddit', 'subreddit_id', 'name', 'created_utc']\n",
    "\n",
    "    file_size = os.stat(input_file_path).st_size\n",
    "    file_lines = 0\n",
    "    file_bytes_processed = 0\n",
    "    bad_lines = 0\n",
    "\n",
    "    with open(output_file_path, \"w\", encoding='utf-8', newline=\"\") as output_file:\n",
    "        writer = csv.writer(output_file)\n",
    "        writer.writerow(fields)  # Write the header\n",
    "\n",
    "        for line, file_bytes_processed in read_lines_zst(input_file_path):\n",
    "            try:\n",
    "                obj = json.loads(line)\n",
    "                if obj.get('subreddit').lower() in (sub.lower() for sub in subreddits):\n",
    "                    writer.writerow([str(obj.get(field, '')).replace(\"\\n\", \" \") for field in fields])\n",
    "            except json.JSONDecodeError as err:\n",
    "                bad_lines += 1\n",
    "            except KeyError as err:\n",
    "                log.info(f\"Missing field in line: {err}\")\n",
    "                bad_lines += 1\n",
    "\n",
    "            file_lines += 1\n",
    "            if file_lines % 100000 == 0:\n",
    "                log.info(f\"Processed {file_lines:,} lines: {bad_lines:,} bad lines : {(file_bytes_processed / file_size) * 100:.2f}% complete\")\n",
    "\n",
    "    log.info(f\"Complete : {file_lines:,} lines processed with {bad_lines:,} bad lines.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "test_data = pd.read_csv('data/comments/comments_2021-11.csv', nrows=1000)\n",
    "test_sub_data = pd.read_csv('reddit/submissions/RS_2021-11.csv', nrows=1000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data.to_csv('data/comments/comments_2021-11_min.csv', index=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sub_data.to_csv('reddit/submissions/submissions_2021-11_min.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "subreddits = ['NOLA', 'acadiana', 'centrallouisiana', 'houma', 'louisiana', \n",
    "                  'batonrouge', 'neworleans', 'Thibodaux', 'newyork', 'Adirondacks', \n",
    "                  'bronx', 'Brooklyn', 'Bushwick', 'hudsonvalley', 'OrangeCountyNY', \n",
    "                  'Queens', 'Westchester', 'Buffalo', 'ForestHills', 'longisland', \n",
    "                  'nyc', 'parkslope', 'williamsburg', 'newjersey', 'Hoboken', \n",
    "                  'jerseycity', 'Newark', 'SouthJersey', 'asburypark', 'NewBrunswickNJ', \n",
    "                  'texas', 'Austin', 'CorpusChristi', 'galveston', 'pflugerville', \n",
    "                  'sanantonio', 'corpus', 'Denton', 'houston']\n",
    "\n",
    "# all items of subreddits list are converted to lower case\n",
    "subreddits = [item.lower() for item in subreddits]\n",
    "\n",
    "# write the updated list to a text file\n",
    "with open('subreddits.txt', 'w') as filehandle:\n",
    "    for listitem in subreddits:\n",
    "        filehandle.write('%s\\n' % listitem)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}