App Engine Task Queue example using pull queues.

Copied from: https://github.com/GoogleCloudPlatform/appengine-pullqueue-counter and tests added.
GoogleCloudPlatform · Apr 12, 2016 · 718417d · 718417d
1 parent 507a179
commit 718417d
Show file tree

Hide file tree

Showing 8 changed files with 169 additions and 0 deletions.
diff --git a/appengine/taskqueue/counter/main_test.py → appengine/taskqueue/counter/counter_test.py b/appengine/taskqueue/counter/main_test.py → appengine/taskqueue/counter/counter_test.py
diff --git a/appengine/taskqueue/pull-counter/README.md b/appengine/taskqueue/pull-counter/README.md
@@ -0,0 +1,8 @@
+# App Engine Task Queue Pull Counter
+
+<!-- auto-doc-link -->
+These samples are used on the following documentation page:
+
+> https://cloud.google.com/appengine/docs/python/taskqueue/overview-pull
+
+<!-- end-auto-doc-link -->
diff --git a/appengine/taskqueue/pull-counter/app.yaml b/appengine/taskqueue/pull-counter/app.yaml
@@ -0,0 +1,11 @@
+runtime: python27
+api_version: 1
+threadsafe: true
+
+handlers:
+- url: /.*
+  script: main.app
+
+libraries:
+- name: jinja2
+  version: 2.6
diff --git a/appengine/taskqueue/pull-counter/counter.html b/appengine/taskqueue/pull-counter/counter.html
@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+<body>
+<form action="/" method="POST">
+  <label for="key">Key:</label><input type="text" name="key" id="key">
+  <input type="submit" value="+1">
+</form>
+<ul>
+{% for counter in counters %}
+<li>
+  {{counter.key.id()|e}}: {{counter.count|e}}
+</li>
+{% endfor %}
+</body>
+</html>
diff --git a/appengine/taskqueue/pull-counter/main.py b/appengine/taskqueue/pull-counter/main.py
@@ -0,0 +1,84 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START all]
+"""A simple counter with App Engine pull queue."""
+
+import logging
+import os
+import time
+
+from google.appengine.api import taskqueue
+from google.appengine.ext import ndb
+from google.appengine.runtime import apiproxy_errors
+import jinja2
+import webapp2
+
+
+JINJA_ENV = jinja2.Environment(
+    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))
+
+
+class Counter(ndb.Model):
+    count = ndb.IntegerProperty(indexed=False)
+
+
+class CounterHandler(webapp2.RequestHandler):
+    def get(self):
+        template_values = {'counters': Counter.query()}
+        counter_template = JINJA_ENV.get_template('counter.html')
+        self.response.out.write(counter_template.render(template_values))
+
+    def post(self):
+        key = self.request.get('key')
+        if key:
+            queue = taskqueue.Queue('pullq')
+            queue.add(taskqueue.Task(payload='', method='PULL', tag=key))
+        self.redirect('/')
+
+
+class CounterWorker(webapp2.RequestHandler):
+    def get(self):
+        """Indefinitely fetch tasks and update the datastore."""
+        queue = taskqueue.Queue('pullq')
+        while True:
+            try:
+                tasks = queue.lease_tasks_by_tag(3600, 1000, deadline=60)
+            except (taskqueue.TransientError,
+                    apiproxy_errors.DeadlineExceededError) as e:
+                logging.exception(e)
+                time.sleep(1)
+                continue
+            if tasks:
+                key = tasks[0].tag
+
+                @ndb.transactional
+                def update_counter():
+                    counter = Counter.get_or_insert(key, count=0)
+                    counter.count += len(tasks)
+                    counter.put()
+                try:
+                    update_counter()
+                except Exception as e:
+                    logging.exception(e)
+                else:
+                    queue.delete_tasks(tasks)
+            time.sleep(1)
+
+
+app = webapp2.WSGIApplication([
+    ('/', CounterHandler),
+    ('/_ah/start', CounterWorker)
+], debug=True)
+# [END all]
diff --git a/appengine/taskqueue/pull-counter/pullcounter_test.py b/appengine/taskqueue/pull-counter/pullcounter_test.py
@@ -0,0 +1,33 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from google.appengine.ext import testbed as gaetestbed
+import main
+import webtest
+
+
+def test_app(testbed):
+    key_name = 'foo'
+
+    testbed.init_taskqueue_stub(root_path=os.path.dirname(__file__))
+
+    app = webtest.TestApp(main.app)
+    app.post('/', {'key': key_name})
+
+    tq_stub = testbed.get_stub(gaetestbed.TASKQUEUE_SERVICE_NAME)
+    tasks = tq_stub.get_filtered_tasks()
+    assert len(tasks) == 1
+    assert tasks[0].name == 'task1'
diff --git a/appengine/taskqueue/pull-counter/queue.yaml b/appengine/taskqueue/pull-counter/queue.yaml
@@ -0,0 +1,3 @@
+queue:
+- name: pullq
+  mode: pull
diff --git a/appengine/taskqueue/pull-counter/worker.yaml b/appengine/taskqueue/pull-counter/worker.yaml
@@ -0,0 +1,15 @@
+module: worker
+api_version: 1
+runtime: python27
+instance_class: B1
+threadsafe: yes
+manual_scaling:
+  instances: 1
+
+handlers:
+- url: /.*
+  script: main.app
+
+libraries:
+- name: jinja2
+  version: 2.6