下载语料的文件名没有格式后缀

.gitignore

@@ -0,0 +1,19 @@
+# maven
+chatbotv1/target/
+
+# py
+__pycache__/
+*.pyc
+
+# vim
+*.swp
+*.swo
+
+# idea
+*.iml
+*.ipr
+*.iws
+.idea/
+
+# tmp files
+subtitle/subtitle_crawler/result/

subtitle/README.md

@@ -0,0 +1,16 @@
+# 爬取字幕语料
+
+* 依赖curl
+* 不支持 Windows
+* Tested with py2.7
+
+## Install
+```
+pip install scrapy
+```
+
+## Download Data
+```
+cd ChatBotCourse/subtitle/subtitle_crawler
+scrapy crawl subtitle
+```

subtitle/subtitle_crawler/pipelines.py

@@ -4,14 +4,17 @@
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
+import os
+from subprocess import call
 
+DOWNLOAD_CMD = os.path.join(os.path.dirname(
+    os.path.realpath(__file__)), 'save.sh')
 
 class SubtitleCrawlerPipeline(object):
+    '''
+    Download Subtitle with curl
+    '''
     def process_item(self, item, spider):
         url = item['url']
-        file_name = url.replace('/','_').replace(':','_')
-        fp = open('result/'+file_name, 'w')
-        fp.write(item['body'])
-        fp.close()
+        call([DOWNLOAD_CMD, url])
         return item
-

subtitle/subtitle_crawler/save.sh

@@ -0,0 +1,19 @@
+#! /bin/bash 
+###########################################
+# Save file with CURL
+###########################################
+
+# constants
+baseDir=$(cd `dirname "$0"`;pwd)
+SAVE_PATH=$baseDir/result
+# functions
+
+# main 
+[ -z "${BASH_SOURCE[0]}" -o "${BASH_SOURCE[0]}" = "$0" ] || return
+if [ ! -d "$SAVE_PATH" ]; then
+  # Control will enter here if $DIRECTORY doesn't exist.
+  mkdir -p $SAVE_PATH
+fi
+
+cd $SAVE_PATH
+curl -JLO $1