MedCode/MedRequests.py at master · amrosado/MedCode · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
__author__ = 'arosado'

import requests
import pymongo
from datetime import datetime, timedelta
from urllib.parse import urlparse, urlencode, urlunparse


class MedRequests:
    requestsSession = None
    mongoClient = None
    requestsDb = None

    def getSessionRequest(self, url, queryDic):
        try:
            #self.requestsDb.drop_collection('wikiRequests')
            if queryDic != None:
                encodedQuery = urlencode(queryDic)
                parsedUrl = urlparse(url+'?'+encodedQuery)
            else:
                parsedUrl = urlparse(url)

            if parsedUrl[1] == 'en.wikipedia.org':
                requestCollection = self.requestsDb.get_collection('wikiRequests')

                requestContent = self.findOrUpdateQueryRequestsDb(requestCollection, parsedUrl)
                return requestContent

        except:
            print('Failed to generate session request')

    def findOrUpdateQueryRequestsDb(self, requestCollection, parsedUrl):
        requestUrl = parsedUrl.geturl()
        urlLocation = parsedUrl[1]
        urlPath = parsedUrl[2]
        urlParams = parsedUrl[3]
        urlQuery = parsedUrl[4]
        urlFragment = parsedUrl[5]

        requestDbQuery = requestCollection.find_one({'urlQuery': urlQuery, 'urlPath': urlPath, 'urlParams': urlParams,\
                                                     'urlFragment':urlFragment, 'urlLocation': urlLocation})
        currentTime = datetime.utcnow()

        if requestDbQuery != None:
            if requestDbQuery['status_code'] == 200:
                requestDatetime = requestDbQuery['datetime']
                timeWeek = requestDatetime + timedelta(days=7)
                if timeWeek > currentTime:
                    return requestDbQuery['content']
                else:
                    sessionRequest = self.requestsSession.get(requestUrl)
                    if sessionRequest.status_code == 200:
                        requestCollection.replace_one({'urlQuery': urlQuery, 'urlPath': urlPath, 'urlParams': urlParams,\
                                'urlFragment':urlFragment, 'urlLocation': urlLocation}, {'urlQuery': urlQuery,\
                                'urlPath': urlPath, 'urlParams': urlParams, 'urlFragment':urlFragment,\
                                'urlLocation': urlLocation, 'content': sessionRequest.content,\
                                'status_code': sessionRequest.status_code, 'datetime': currentTime})
                        return sessionRequest.content
            else:
                sessionRequest = self.requestsSession.get(requestUrl)
                if sessionRequest.status_code == 200:
                    requestCollection.replace_one({'urlQuery': urlQuery, 'urlPath': urlPath, 'urlParams': urlParams,\
                        'urlFragment':urlFragment, 'urlLocation': urlLocation}, {'urlQuery': urlQuery,\
                        'urlPath': urlPath, 'urlParams': urlParams, 'urlFragment':urlFragment,\
                        'urlLocation': urlLocation, 'content': sessionRequest.content,\
                        'status_code': sessionRequest.status_code, 'datetime': currentTime})
                    return sessionRequest.content
                else:
                    raise Exception('Error with request')
        else:
            sessionRequest = self.requestsSession.get(requestUrl)
            requestCollection.insert({'urlQuery': urlQuery,\
                    'urlPath': urlPath, 'urlParams': urlParams, 'urlFragment':urlFragment,\
                    'urlLocation': urlLocation, 'content': sessionRequest.content,\
                    'status_code': sessionRequest.status_code, 'datetime': currentTime})
            if sessionRequest.status_code == 200:
                return sessionRequest.content
            else:
                raise Exception('Error with request')

    def saveSessionRequest(self, request):
        pass

    def __init__(self):
        try:
            self.requestsSession = requests.Session()

            self.mongoClient = pymongo.MongoClient()
            self.requestsDb = self.mongoClient.get_database('RequestsDatabase')

        except:
            print('Failed to initialize Med Requests object.  Check database connection')