import tornado
from tornado import httpclient
from tornado import gen
from functools import partial
import Queue
from tornado.httpclient import AsyncHTTPClient
import os
from time import gmtime, strftime
import json
from urlparse import urlparse

gloop = tornado.ioloop.IOLoop.instance()
qinput = Queue.Queue()
process_count = 0

process_max = 15000    # maximum count of query for one moment

#create tmp dir if not exists
dirname = "tmp"
if not os.path.exists(dirname):
    os.makedirs(dirname)

#fill queue
f = open('100000_hostsList.lst')
line = f.readline()
items = 0
hosts = []
while line:
    qinput.put("http://"+line)
    line = f.readline()

f.close()


def data_process(data, url, headers):
    data = {'url': url, 'data': data, 'headers': headers}

    dirname = "tmp/" + strftime("%Y-%m-%d_%H", gmtime())
    if not os.path.exists(dirname):
        os.makedirs(dirname)

    f = file(dirname + "/" + urlparse(url).hostname, "w+")
    f.write(json.dumps(data))
    f.flush()
    f.close()

@gen.engine
def process(url):
    global process_count, worker
    try:
        http_client = httpclient.AsyncHTTPClient()

        request = tornado.httpclient.HTTPRequest(url=str(url), connect_timeout=5.0, request_timeout=5.0, follow_redirects=True)
        response = yield tornado.gen.Task(http_client.fetch, request)

        if response.error: raise Exception(response.error)
        data_process(response.body, url, dict(response.headers))
    except Exception as e:
        print e
    process_count -= 1
    gloop.add_callback(worker)

def worker():
    global gloop, process_count, process_max
    print '# %d / %d (%d)' % (process_count, process_max, qinput.qsize())
    while process_count < process_max:
        if qinput.empty(): break
        url = qinput.get_nowait()
        process_count += 1
        gloop.add_callback(partial(process, url))
    if qinput.empty():
        if not process_count: gloop.stop()

print 'start'
gloop.add_callback(worker)
tornado.httpclient.AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient", max_clients=100)
gloop.start()
print 'finish'