1_count_rows_per_machines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "44cf4336-5b3d-4f20-a2e8-47fa7b9de11a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   machine_id  mean_cpu_usage_rate  maximum_cpu_usage  sampled_cpu_usage\n",
      "0  4820204869              0.03143             0.2415            0.02457\n",
      "1  3349189123              0.02698             0.2700            0.01608\n",
      "2   372630265              0.04114             0.1086            0.05286\n",
      "3     2850406              0.00000             0.0000            0.00000\n",
      "4  1437119225              0.07275             0.2415            0.06238\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import gzip\n",
    "\n",
    "# Task Usage Table\n",
    "column_names = [  \n",
    "    'start_time',   \n",
    "    'end_time',   \n",
    "    'job_id',   \n",
    "    'task_index',   \n",
    "    'machine_id',   \n",
    "    'mean_cpu_usage_rate',  \n",
    "    'canonical_memory_usage',   \n",
    "    'assigned_memory_usage',   \n",
    "    'unmapped_page_cache_memory_usage',  \n",
    "    'total_page_cache_memory_usage',   \n",
    "    'maximum_memory_usage',   \n",
    "    'mean_disk_io_time',  \n",
    "    'mean_local_disk_space_used',   \n",
    "    'maximum_cpu_usage',   \n",
    "    'maximum_disk_io_time',   \n",
    "    'cpi',   \n",
    "    'mai',  \n",
    "    'sample_portion',   \n",
    "    'aggregation_type',   \n",
    "    'sampled_cpu_usage'  \n",
    "]   \n",
    "\n",
    "file_path = 'clusterdata-2011-2/task_usage/part-00001-of-00500.csv.gz'\n",
    "df = pd.read_csv(file_path, compression='gzip', header=None, names=column_names)\n",
    "df = df[['machine_id', 'mean_cpu_usage_rate', 'maximum_cpu_usage', 'sampled_cpu_usage']] \n",
    "\n",
    "print(df.head(5))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "aa1e0fac-4d73-4aa5-bb7b-e825d5d62c0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing clusterdata-2011-2/task_usage/part-00000-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00001-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00002-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00003-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00004-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00005-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00006-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00007-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00008-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00009-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00010-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00011-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00012-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00013-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00014-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00015-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00016-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00017-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00018-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00019-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00020-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00021-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00022-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00023-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00024-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00025-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00026-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00027-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00028-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00029-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00030-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00031-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00032-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00033-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00034-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00035-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00036-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00037-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00038-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00039-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00040-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00041-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00042-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00043-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00044-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00045-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00046-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00047-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00048-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00049-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00050-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00051-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00052-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00053-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00054-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00055-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00056-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00057-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00058-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00059-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00060-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00061-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00062-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00063-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00064-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00065-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00066-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00067-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00068-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00069-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00070-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00071-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00072-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00073-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00074-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00075-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00076-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00077-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00078-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00079-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00080-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00081-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00082-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00083-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00084-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00085-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00086-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00087-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00088-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00089-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00090-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00091-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00092-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00093-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00094-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00095-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00096-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00097-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00098-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00099-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00100-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00101-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00102-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00103-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00104-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00105-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00106-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00107-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00108-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00109-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00110-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00111-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00112-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00113-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00114-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00115-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00116-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00117-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00118-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00119-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00120-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00121-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00122-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00123-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00124-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00125-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00126-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00127-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00128-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00129-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00130-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00131-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00132-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00133-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00134-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00135-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00136-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00137-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00138-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00139-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00140-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00141-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00142-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00143-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00144-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00145-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00146-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00147-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00148-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00149-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00150-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00151-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00152-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00153-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00154-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00155-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00156-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00157-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00158-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00159-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00160-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00161-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00162-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00163-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00164-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00165-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00166-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00167-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00168-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00169-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00170-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00171-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00172-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00173-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00174-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00175-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00176-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00177-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00178-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00179-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00180-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00181-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00182-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00183-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00184-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00185-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00186-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00187-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00188-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00189-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00190-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00191-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00192-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00193-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00194-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00195-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00196-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00197-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00198-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00199-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00200-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00201-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00202-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00203-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00204-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00205-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00206-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00207-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00208-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00209-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00210-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00211-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00212-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00213-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00214-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00215-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00216-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00217-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00218-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00219-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00220-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00221-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00222-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00223-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00224-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00225-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00226-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00227-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00228-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00229-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00230-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00231-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00232-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00233-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00234-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00235-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00236-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00237-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00238-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00239-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00240-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00241-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00242-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00243-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00244-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00245-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00246-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00247-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00248-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00249-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00250-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00251-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00252-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00253-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00254-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00255-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00256-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00257-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00258-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00259-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00260-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00261-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00262-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00263-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00264-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00265-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00266-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00267-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00268-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00269-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00270-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00271-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00272-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00273-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00274-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00275-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00276-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00277-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00278-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00279-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00280-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00281-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00282-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00283-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00284-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00285-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00286-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00287-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00288-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00289-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00290-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00291-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00292-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00293-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00294-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00295-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00296-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00297-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00298-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00299-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00300-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00301-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00302-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00303-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00304-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00305-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00306-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00307-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00308-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00309-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00310-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00311-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00312-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00313-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00314-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00315-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00316-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00317-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00318-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00319-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00320-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00321-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00322-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00323-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00324-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00325-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00326-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00327-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00328-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00329-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00330-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00331-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00332-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00333-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00334-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00335-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00336-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00337-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00338-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00339-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00340-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00341-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00342-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00343-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00344-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00345-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00346-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00347-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00348-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00349-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00350-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00351-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00352-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00353-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00354-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00355-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00356-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00357-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00358-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00359-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00360-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00361-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00362-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00363-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00364-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00365-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00366-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00367-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00368-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00369-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00370-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00371-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00372-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00373-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00374-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00375-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00376-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00377-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00378-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00379-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00380-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00381-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00382-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00383-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00384-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00385-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00386-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00387-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00388-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00389-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00390-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00391-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00392-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00393-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00394-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00395-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00396-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00397-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00398-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00399-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00400-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00401-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00402-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00403-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00404-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00405-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00406-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00407-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00408-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00409-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00410-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00411-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00412-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00413-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00414-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00415-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00416-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00417-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00418-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00419-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00420-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00421-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00422-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00423-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00424-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00425-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00426-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00427-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00428-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00429-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00430-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00431-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00432-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00433-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00434-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00435-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00436-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00437-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00438-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00439-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00440-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00441-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00442-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00443-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00444-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00445-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00446-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00447-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00448-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00449-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00450-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00451-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00452-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00453-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00454-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00455-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00456-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00457-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00458-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00459-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00460-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00461-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00462-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00463-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00464-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00465-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00466-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00467-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00468-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00469-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00470-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00471-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00472-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00473-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00474-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00475-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00476-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00477-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00478-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00479-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00480-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00481-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00482-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00483-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00484-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00485-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00486-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00487-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00488-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00489-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00490-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00491-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00492-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00493-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00494-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00495-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00496-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00497-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00498-of-00500.csv.gz\n",
      "processing clusterdata-2011-2/task_usage/part-00499-of-00500.csv.gz\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# Initialize an empty DataFrame to store the counts  \n",
    "machine_counts = pd.DataFrame()  \n",
    "  \n",
    "# Loop over all files  \n",
    "for i in range(500):  \n",
    "    file_path = f'clusterdata-2011-2/task_usage/part-{str(i).zfill(5)}-of-00500.csv.gz'  \n",
    "    print(\"processing \" + file_path)\n",
    "    if os.path.exists(file_path):  \n",
    "        df = pd.read_csv(file_path, compression='gzip', header=None, names=column_names)  \n",
    "  \n",
    "        # Count the number of rows for each machine_id  \n",
    "        counts = df['machine_id'].value_counts().reset_index()  \n",
    "        counts.columns = ['machine_id', 'rows']  \n",
    "  \n",
    "        # Add the counts to the total  \n",
    "        machine_counts = pd.concat([machine_counts, counts])  \n",
    "  \n",
    "# Group by machine_id and sum the counts  \n",
    "machine_counts = machine_counts.groupby('machine_id').sum().reset_index()  \n",
    "machine_counts = machine_counts.sort_values(by='rows', ascending=False)\n",
    "\n",
    "machine_counts.to_csv('machine_taskrows_counts.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23a84308-26aa-4ed9-b16b-1a296bd6b149",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}