Skip to content

Commit

Permalink
removed test timeout
Browse files Browse the repository at this point in the history
  • Loading branch information
@BorisTherin
BorisTherin committed Apr 7, 2024
1 parent 6f75d2c commit b332de2
Show file tree
Hide file tree
Showing 2 changed files with 354 additions and 1 deletion.
353 changes: 353 additions & 0 deletions notebook/ingest.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,353 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "6e5803c7-95c2-4cf7-b845-0c12292e7311",
"metadata": {},
"source": [
"## import de notre package"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "30f104be-d566-4287-985e-e4b8add86f31",
"metadata": {},
"outputs": [],
"source": [
"import { ingesters, transformers, utils } from 'npm:@decoder-leco/poc-eurostat-data-transformers@0.0.1-gamma-commit-0ea0eb1c5be6aefee96b41e0800673a236fa15b3'"
]
},
{
"cell_type": "markdown",
"id": "ce86bd53-5dfb-442f-8383-b06a6ad3220b",
"metadata": {},
"source": [
"### ingestion avec la classe github"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "4629f0f6-14b3-4dd5-96f8-1894f37e7e2c",
"metadata": {},
"outputs": [],
"source": [
"const populationProjection_2019_2024_Ingester = new ingesters.DecoderLecoGithubDataIngester(\n",
" \"main\",\n",
" \"data/csv/proj_19np__custom_2224172_linear.csv\",\n",
" `./data_pipeline_workdir/42`\n",
");"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a1dce2cc-ee0a-4d00-aa91-e93361fb8f60",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"dir: Skipped creating the [./data_pipeline_workdir/42/data/csv] folder, because it already exists.\n",
"DL: [main] version of File data/csv/proj_19np__custom_2224172_linear.csv has succesfully been downloaded from [https://github.com/decoderleco/deces_europe.git]\n"
]
}
],
"source": [
"const [ logDir, logDL ] = await populationProjection_2019_2024_Ingester.run()\n",
"console.log('dir: '+logDir)\n",
"console.log('DL: '+logDL)"
]
},
{
"cell_type": "markdown",
"id": "9b4920c5-9b5b-43bc-a6b5-fa72173cf4c0",
"metadata": {},
"source": [
"### ingestions avec la classe eurostat"
]
},
{
"cell_type": "markdown",
"id": "9b19ac17-9118-4132-a7e7-77e97da75e8b",
"metadata": {},
"source": [
"`demo_pjan`"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "fe0aa7c1-7384-4e68-9e0f-d0e4b2b0ba19",
"metadata": {},
"outputs": [],
"source": [
"const pjan = new ingesters.DecoderLecoEurostatDataIngester(\n",
" 'demo_pjan',\n",
" 'SDMX-CSV', \n",
" './data_pipeline_workdir/eurostat'\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "7b2e61e8-6338-410e-a5d8-6445a3714f84",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"dir: Skipped creating the [./data_pipeline_workdir/eurostat] folder, because it already exists.\n",
"DL: File ./data_pipeline_workdir/eurostat/demo_pjan.csv has succesfully been writed\n"
]
}
],
"source": [
"const [ logDir, logDL ] = await pjan.run()\n",
"console.log('dir: '+logDir)\n",
"console.log('DL: '+logDL)\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "08df98b0-e46d-433f-aee8-3263660c9ae3",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[32m\"demo_pjan.csv\"\u001b[39m"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pjan.getIngestedDataFileName()"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "cdc88a01-eaeb-4825-b10f-fea8d46b9dad",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[32m\"./data_pipeline_workdir/eurostat\"\u001b[39m"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pjan.getIngestedDataFileFolderPath()"
]
},
{
"cell_type": "markdown",
"id": "83a0989e-91d2-4a8b-b74f-f409fa1ace00",
"metadata": {},
"source": [
"`demo_magec`"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "6f71c373-698d-4e78-b7ce-29ad8692aca5",
"metadata": {},
"outputs": [],
"source": [
"const magec = new ingesters.DecoderLecoEurostatDataIngester(\n",
" 'demo_magec',\n",
" 'SDMX-CSV', \n",
" './data_pipeline_workdir/eurostat'\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "5bc34b67-adc8-4559-b4a6-7587b338b6cd",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"dir: Skipped creating the [./data_pipeline_workdir/eurostat] folder, because it already exists.\n",
"DL: File ./data_pipeline_workdir/eurostat/demo_magec.csv has succesfully been writed\n"
]
}
],
"source": [
"const [ logDir, logDL ] = await magec.run()\n",
"console.log('dir: '+logDir)\n",
"console.log('DL: '+logDL)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "bf5846a6-9504-4cab-aa8f-d3541dc10f2b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[32m\"demo_magec.csv\"\u001b[39m"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"magec.getIngestedDataFileName()"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "913c8514-ddb2-444d-9e8b-410e7118cd68",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[32m\"./data_pipeline_workdir/eurostat\"\u001b[39m"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"magec.getIngestedDataFileFolderPath()"
]
},
{
"cell_type": "markdown",
"id": "22710a27-46b2-4f68-ba35-b218d520e2cb",
"metadata": {},
"source": [
"`demo_r_mwk_05`"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "9f57e6eb-bd49-4e2f-a278-8e07c31e07af",
"metadata": {},
"outputs": [],
"source": [
"const mwk = new ingesters.DecoderLecoEurostatDataIngester(\n",
" 'demo_r_mwk_05',\n",
" 'SDMX-CSV', \n",
" './data_pipeline_workdir/eurostat'\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "df28632b-1577-4f55-932b-fea51a6cca26",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"dir: Skipped creating the [./data_pipeline_workdir/eurostat] folder, because it already exists.\n",
"DL: File ./data_pipeline_workdir/eurostat/demo_r_mwk_05.csv has succesfully been writed\n"
]
}
],
"source": [
"const [ logDir, logDL ] = await mwk.run()\n",
"console.log('dir: '+logDir)\n",
"console.log('DL: '+logDL)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "3212c2e0-c580-49be-af44-3453a5322df7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[32m\"demo_r_mwk_05.csv\"\u001b[39m"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mwk.getIngestedDataFileName()"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "5f190a5d-8a79-47a7-9336-66cd4381c73f",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[32m\"./data_pipeline_workdir/eurostat\"\u001b[39m"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mwk.getIngestedDataFileFolderPath()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9171ddfa-4807-42ca-a3c9-2a5b85a3449f",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Deno",
"language": "typescript",
"name": "deno"
},
"language_info": {
"file_extension": ".ts",
"mimetype": "text/x.typescript",
"name": "typescript",
"nb_converter": "script",
"pygments_lexer": "typescript",
"version": "5.3.3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
2 changes: 1 addition & 1 deletion tests/ingesters/DecoderLecoEurostatDataIngester.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,6 @@ describe('Testing - DecoderLecoGithubDataIngester', () => {
expect(fs.readFileSync(`${testDataWorkDir}/${testFilePathInEurostat}.csv`, { encoding: 'utf8', flag: 'r' }) == "404 error" ).toBe(false)
},
// large timeout for large files
500000)
)
})
})

0 comments on commit b332de2

Please sign in to comment.