ability to recieve data from the frontend with the user aggregations

gamechangerml/api/fastapi/routers/controls.py

@@ -973,21 +973,20 @@ async def stop_process(thread_dict: dict, response: Response):
 
  return {'stopped':thread_id}
 
-@router.post("/getUserData")
-async def stop_process(date_dict: dict, response: Response):
- """Get user aggregation data for selected date and write to data folder
+@router.post("/sendUserAggregations")
+async def get_user_data(data_dict: dict, response: Response):
+ """get_user_data - Get user aggregation data for selected date and write to data folder
  Args:
- date_dict: dict; {startDate,endDate}
+ date_dict: dict; {data}
  Response: Response class; for status codes(apart of fastapi do not need to pass param)
  Returns:
- Stopped thread id
+ confirmation of data download
  """
-
- data = json.loads(gcClient.getUserAggregations(start_date=date_dict['startDate'],end_date=date_dict['endDate']))
+ data = data_dict['params']['data']
  GC_USER_DATA = os.path.join(
  DATA_PATH, "user_data", "UserAggregations.json"
  )
  with open(GC_USER_DATA,'w') as f:
  json.dump(data,f)
 
- return 'wrote user data to file'
+ return f'wrote {len(data)} user data to file'

gamechangerml/data/user_data/UserAggregations.json

@@ -1 +1 @@
-{"users": [{"idvisitor": "031A24EEF605BC74", "docs_opened": 0, "searches_made": 2, "opened": [], "export": [], "favorite": []}, {"idvisitor": "1CEAB3D67A91BFA1", "docs_opened": 34, "searches_made": 56, "opened": ["S.Res. 536 IS 117th.pdf", "MIL-D-23222 Revision A Notice 1", "OPNAVINST 3100.8B.pdf", "MIL-C-2683 Revision A Notice 4", "H.R. 5467 IH 117th.pdf"], "export": [], "favorite": ["MIL-D-23222 Revision A Notice 1 - Validation 2019-06-25.pdf", "OPNAVINST 3100.8B.pdf", "DoDD 5136.01 CH 1.pdf"]}, {"idvisitor": "23BCE26849B77287", "docs_opened": 5, "searches_made": 1, "opened": ["DoD Data Strategy.PD[\u2026]geNumber=0"], "export": [], "favorite": []}, {"idvisitor": "58751A4353CBF7C0", "docs_opened": 109, "searches_made": 1087, "opened": ["ARMY DIR 2020-06.pdf", "18-006- .pdf", "MIL-DTL-32541 Base Document 2015-11-30.pdf", "DoDI 1235.12 CH 1.pdf", "6200.01 .pdf"], "export": ["DoDD 5137.02.pdf", "CJCSI 1805.01C.pdf", "DoDI 1215.21 CH 1.pdf", "DoDI 1322.26.pdf", "AI 40.pdf"], "favorite": ["TM 4-48.09.pdf", "STP 9-94K13-SM-TG.pdf", "ATP 3-04.19.pdf", "DoDD 3025.13 CH 1.pdf", "SECNAVINST 5239.24.pdf"]}, {"idvisitor": "60592F05B49DCD68", "docs_opened": 4, "searches_made": 24, "opened": ["DHA Procedural Instructions 6010.01.pdf", "CFR-2021-title26-vol18.pdf"], "export": [], "favorite": ["PGI 239.76 - CLOUD COMPUTING.pdf"]}, {"idvisitor": "62FC9F3443EC3AE3", "docs_opened": 33, "searches_made": 92, "opened": ["Title 9", "DFARS SUBPART 222.74", "PGI 222.74", "SECNAVINST 5800.15A.pdf", "SECNAVINST 3300.2C.pdf"], "export": [], "favorite": ["SECNAVINST 5239.24.pdf", "COMNAVRESFORCOMINST 5726.3.pdf", "OPNAVINST 5420.106A.pdf"]}, {"idvisitor": "64DC6ABB20F9B2C3", "docs_opened": 1, "searches_made": 5, "opened": ["H.R. 5467 IH 117th.pdf"], "export": [], "favorite": ["SECNAVINST 5239.24.pdf", "H.R. 5467 IH 117th.pdf", "DoDD 3200.15 CH 1.pdf"]}, {"idvisitor": "6A4EE43B3AFF62EB", "docs_opened": 0, "searches_made": 2, "opened": [], "export": [], "favorite": []}, {"idvisitor": "71339E3B3F85E3B4", "docs_opened": 240, "searches_made": 1735, "opened": ["CI 16601.28B.pdf", "DoDD 5105.19.pdf", "CIM 16000.14A.pdf", "ATP 97.pdf", "SASC Hearing"], "export": ["AI 40.pdf", "DoDD 5137.02.pdf", "CJCSI 1805.01C.pdf", "Title 15 - Commerce and Trade.pdf", "CNSSI 4009.pdf"], "favorite": ["ARMY 2018-18 ARMY ARTIFICIAL INTELLIGENCE TASK FORCE IN SUPPORT OF THE DEPARTMENT OF DEFENSE JOINT ARTIFICIAL INTELLIGENCE CENTER", "Title 10 Armed Forces", "Department of Defense Handbook MIL-HDBK-57D Listing of Fastener Manufacturer's Identification Symbols", "EO 13822 Supporting Our Veterans During Their Transition From Uniformed Service to Civilian Life", "H.R. 922 To prohibit United States Government recognition of the Russian Federation\u2019s claim of sovereignty over Crimea, and for other purposes."]}, {"idvisitor": "721E722041A6EC2D", "docs_opened": 0, "searches_made": 6, "opened": [], "export": [], "favorite": []}, {"idvisitor": "7976822F1CDC4562", "docs_opened": 35, "searches_made": 22, "opened": ["DoDM 4140.01 Volume 9 CH 2.pdf", "DoD Data Strategy.PD[\u2026]geNumber=0", "DoD Data Strategy.PDF", "H.R. 5467 IH 117th.pdf", "H 5467 IH 117th.pdf"], "export": [], "favorite": []}, {"idvisitor": "7F06C04D59BD8360", "docs_opened": 12, "searches_made": 57, "opened": ["DoDI 5240.19 CH 2.pdf", "AFI 10-202.pdf", "ARMY DIR 2021-35.pdf", "MCO 1730.8.pdf"], "export": [], "favorite": []}, {"idvisitor": "825777C330FD094B", "docs_opened": 0, "searches_made": 28, "opened": [], "export": [], "favorite": []}, {"idvisitor": "8E0BCAF48C3D73DD", "docs_opened": 288, "searches_made": 453, "opened": ["STIG V1R1 Riverbed SteelHead CX v8 STIG Ver 1 Release Memo.pdf", "STIG V0R0 Vendor STIG Acronym List.pdf", "STIG V0R0 CCI Process.pdf", "STIG V0R0 2016-04-21 DoD CIO Memo", "STANAG 4439.pdf"], "export": ["OO-S-256_11 Revision B Notice 2 - Validation 2006-11-01.pdf", "OO-S-256_11 Revision B Notice 3 - Validation 2020-02-05.pdf", "OO-S-256_1 Revision B Notice 3 - Validation 2020-02-04.pdf", "MIL-G-58103 Notice 3 - Validation 2010-03-23.pdf", "MIL-G-58103 Notice 4 - Validation 2014-12-23.pdf"], "favorite": ["3020.01,-Change-2 .pdf", "MIL-R-8592 Revision B Notice 2 - Validation 2019-08-22.pdf", "H.R. 5467 IH 117th.pdf"]}, {"idvisitor": "9160B3FFCCAD442E", "docs_opened": 2, "searches_made": 18, "opened": ["DoDI 5015.02 CH 1.pdf"], "export": [], "favorite": []}, {"idvisitor": "922C762749F5451C", "docs_opened": 0, "searches_made": 79, "opened": [], "export": [], "favorite": []}, {"idvisitor": "9F6BBEC48D1BBD01", "docs_opened": 0, "searches_made": 31, "opened": [], "export": [], "favorite": []}, {"idvisitor": "AB92B386244A1CE6", "docs_opened": 0, "searches_made": 2, "opened": [], "export": [], "favorite": []}, {"idvisitor": "B102CE1D5EEBAC2B", "docs_opened": 0, "searches_made": 12, "opened": [], "export": [], "favorite": []}, {"idvisitor": "B88667501EF34887", "docs_opened": 0, "searches_made": 1, "opened": [], "export": [], "favorite": []}, {"idvisitor": "CA87275CB32384EA", "docs_opened": 0, "searches_made": 4, "opened": [], "export": [], "favorite": []}, {"idvisitor": "CFE5B0C8CD296892", "docs_opened": 0, "searches_made": 5, "opened": [], "export": [], "favorite": []}, {"idvisitor": "DCF48D0B6FBA15A2", "docs_opened": 7, "searches_made": 21, "opened": ["OMBM M-16-21.pdf", "TC 3-22.50.pdf", "MCTP 3-01C.pdf", "ARMY DIR 2018-07-13.pdf"], "export": [], "favorite": ["OPNAVINST 5420.106A.pdf"]}, {"idvisitor": "DD80B9C66CADA1DA", "docs_opened": 27, "searches_made": 426, "opened": ["MIL-DTL-32541 Base Document 2015-11-30.pdf", "DoDI 1235.12 CH 1.pdf", "012 .pdf", "18-006- .pdf"], "export": ["DoDD 5000.01.pdf", "DoDI 4151.22.pdf", "DoDI 1215.21 CH 1.pdf", "DoDI 1322.26.pdf"], "favorite": []}, {"idvisitor": "E0766A1B3314BBAD", "docs_opened": 3, "searches_made": 2, "opened": ["TM 5-642_dup1.pdf", "ARMY DIR 2020-06.pdf"], "export": [], "favorite": []}, {"idvisitor": "E33C668433DEBFCC", "docs_opened": 26, "searches_made": 136, "opened": ["NAVMC 3500.100C.pdf", "ARMY DIR 2020-06.pdf", "PGI 239.76", "DoDI 1015.10 CH 1.pdf", "MCRP 3-20B.1.pdf"], "export": [], "favorite": []}, {"idvisitor": "E879AA30341B8519", "docs_opened": 28, "searches_made": 137, "opened": ["SECNAVINST 5239.24.pdf", "DFARS SUBPART 239.76", "DFARS SUBPART 239.76.html", "MARADMIN MARADMIN CANCELLATION 011-13 CANCELLATION OF MARADMIN 103-13.html"], "export": [], "favorite": []}, {"idvisitor": "E8996EFD2D75C4CC", "docs_opened": 33, "searches_made": 137, "opened": ["MARADMIN 031-13 CANCELLATION OF MARADMIN 325-13.html", "MARADMIN 036-10 CANCELLATION OF MARADMIN 367-10.html", "MARADMIN 049-19 CANCELLATION OF MARADMIN 628-19.html", "SECNAVINST 5213.16.pdf", "MCO 6320.5.pdf"], "export": [], "favorite": ["SECNAVINST 5239.24.pdf"]}, {"idvisitor": "EA8F8BB9A8A7B95C", "docs_opened": 0, "searches_made": 5, "opened": ["EDAPDF-00AF985C0EC44D12E05400215A9BA3BA-H9222210D0016-0015-empty-01-PDS-2014-08-15.pdf"], "export": [], "favorite": []}, {"idvisitor": "EC088268C2A5CABE", "docs_opened": 7, "searches_made": 13, "opened": ["MIL-T-58100 Notice 4", "MIL-D-23222 Revision A Notice 1", "ICD 700.pdf"], "export": ["DoDI 2000.12 CH 3.pdf"], "favorite": ["SECNAVINST 3300.2C.pdf", "SECNAVINST 5239.24.pdf"]}, {"idvisitor": "EE51D5B9B57CD8A1", "docs_opened": 8, "searches_made": 42, "opened": ["OPNAVINST 5420.106A.pdf", "SECNAVINST 5800.15A.pdf", "H.R. 2812 IH 117th.pdf"], "export": [], "favorite": []}], "cards": {"unique_users": 31, "total_searches": 5543}}
+[{"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "3815f36cc7ee36c3a7eeb2aa65ca9e33", "org": "Advana", "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "098f6bcd4621d373cade4e832627b4f6", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "f2e3e25e63be9acbb82c1e0ba8eabae6", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 86, "user_id": "283f42764da6dba2522412916b031080", "org": null, "last_search": "2022-04-20T21:54:02.000Z", "last_search_formatted": "2022-04-20 21:54"}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "ce4828f7ac5478fab22b83214fa4e9e8", "org": "OSD", "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "c6034f2753944551790abf1e11524f41", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 1, "user_id": "6cc0d36686e6a433aa76f96773852d35", "org": "Advana", "last_search": "2022-04-20T16:25:47.000Z", "last_search_formatted": "2022-04-20 16:25"}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "9642ced8dc364a3d7fadfc2ad4f05767", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "741703dc66a64873edecea7af9a79c36", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 1, "user_id": "acf4b89d3d503d8252c9c4ba75ddbf6d", "org": null, "last_search": "2022-04-20T20:53:19.000Z", "last_search_formatted": "2022-04-20 20:53"}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "17831f414ececf207ffdc2b480276e10", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "c8df92d8d044aeb4d98acd39f37a3b47", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "74728139215254a684fb33cdbe7f2611", "org": "Booz Allen Hamilton", "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "c678b11ff617890420ba49a655c8f954", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "fcea920f7412b5da7be0cf42b8c93759", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": ["ARMY DIR 2020-06.pdf", "MIL-DTL-32541 Base Document 2015-11-30.pdf", "DoDI 1235.12 CH 1.pdf", "6200.01 .pdf"], "ExportDocument": ["DoDI 4151.22.pdf", "DoDD 5000.01.pdf", "DoDD 5137.02.pdf", "CJCSI 1805.01C.pdf", "CNSSI 4009.pdf"], "Favorite": ["ARMY DIR 2020-06.pdf", "TM 4-48.09.pdf", "STP 9-94K13-SM-TG.pdf", "ATP 3-04.19.pdf"], "docs_opened": 28, "searches_made": 286, "user_id": "8c12102acb429ec0e80d8941334eaac8", "org": null, "last_search": "2022-04-21T08:37:26.000Z", "last_search_formatted": "2022-04-21 08:37"}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "992e63080ee1e47b99f42b8d64ede953", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "702263593df9b9a7efa6454e01b04c9b", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 231, "user_id": "63effa2530d088a06f071bc5f016f8d4", "org": "Booz Allen", "last_search": "2022-04-22T15:31:26.000Z", "last_search_formatted": "2022-04-22 15:31"}, {"opened": ["CI 3120.4A.pdf", "OPNAVINST 5420.117.pdf"], "ExportDocument": [], "Favorite": [], "docs_opened": 3, "searches_made": 87, "user_id": "d69403e2673e611d4cbd3fad6fd1788e", "org": null, "last_search": "2022-04-21T18:52:20.000Z", "last_search_formatted": "2022-04-21 18:52"}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "9299a160d5bca374400f96cb72a53206", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "00e0c490b5641899c8792454d7ea3c33", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": ["SECNAVINST 5239.24.pdf"], "ExportDocument": [], "Favorite": [], "docs_opened": 2, "searches_made": 7, "user_id": "6c30734811916b0f0f24a4630b08036f", "org": "Advana", "last_search": "2022-04-21T16:11:59.000Z", "last_search_formatted": "2022-04-21 16:11"}, {"opened": ["EDAPDF-C4A0CCD27A411532E0440025B3E8F0A7-SP070003D1380-0462-empty-empty-PDS-2012-07-12.pdf"], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "1c6e8ca83d72257467b4a3c08e90ed1b", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "c642426069bf4ce696f2f0560cca9419", "org": "bah", "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "dd52ee653a05dfc30b0817732316efed", "org": "Booz Allen", "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "bfd81ee3ed27ad31c95ca75e21365973", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "99ddba2d6575cc7b604143e16f6ba082", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "827ccb0eea8a706c4c34a16891f84e7b", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "e10adc3949ba59abbe56e057f20f883e", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "33ee85af6cd23552b87e7864f5f75960", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "9e94b15ed312fa42232fd87a55db0d39", "org": "OSD", "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "f1c1592588411002af340cbaedd6fc33", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "b63d5bfca2fa5652fefb17cce756a18a", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "c0b9ef40f5d27ec41ad2bb5614ada454", "org": null, "last_search": null, "last_search_formatted": ""}, {"opened": [], "ExportDocument": [], "Favorite": [], "docs_opened": 0, "searches_made": 0, "user_id": "6ff71226872d6896380f09bda226e8a2", "org": "Booz Allen Hamilton", "last_search": null, "last_search_formatted": ""}]

gamechangerml/src/utilities/utils.py

@@ -133,7 +133,6 @@ def get_s3_corpus(s3_corpus_dir, output_dir="corpus"):
  corp = []
  try:
  bucket = s3_connect()
-
  if not os.path.exists(output_dir):
  os.makedirs(output_dir)
  else: