First

qasync_phantompy.py

@@ -0,0 +1,132 @@
+#!/usr/local/bin/python3.sh
+# -*-mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*
+
+import sys
+import os
+import qasync
+import asyncio
+import time
+import random
+
+from PyQt5 import QtWidgets
+from PyQt5.QtWidgets import (QProgressBar, QWidget, QVBoxLayout)
+
+from phantompy import Render
+from lookupdns import LookFor as Render
+
+global LOG
+import logging
+import warnings
+warnings.filterwarnings('ignore')
+LOG = logging.getLogger()
+
+class Widget(QtWidgets.QWidget):
+    def __init__(self):
+        QtWidgets.QWidget.__init__(self)
+        self._label = QtWidgets.QLabel()
+        box = QtWidgets.QHBoxLayout()
+        self.setLayout(box)
+        box.addWidget(self._label)
+        self.progress = QProgressBar()
+        self.progress.setRange(0, 99)
+        box.addWidget(self.progress)
+
+    def update(self, text):
+        i = len(asyncio.all_tasks())
+        self._label.setText(str(i))
+        self.progress.setValue(int(text))
+        
+class ContextManager:
+    def __init__(self) -> None:
+        self._seconds = 0
+    async def __aenter__(self):
+        LOG.debug("ContextManager enter")
+        return self
+    async def __aexit__(self, *args):
+        LOG.debug("ContextManager exit")
+    async def tick(self):
+        await asyncio.sleep(1)
+        self._seconds += 1
+        return self._seconds
+
+async def main(widget, app, ilen):
+    LOG.debug("Task started")
+    try:
+        async with ContextManager() as ctx:
+            for i in range(1, 120):
+                seconds = await ctx.tick()
+                LOG.info(str(seconds))
+                perc = 50 + int(float(len(app.lfps))*100.0/ilen)
+                if widget:
+                    widget.update(str(perc))
+                LOG.debug(f"{app.lfps} {perc} {seconds}")
+                if len(app.lfps) == ilen:
+                    print('\n'.join(app.lfps))
+                    app.exit()
+                    # raise  asyncio.CancelledError
+                    break
+    except asyncio.CancelledError as ex:
+        LOG.debug("Task cancelled")
+
+def iMain(largs, bgui=True):
+    app = QtWidgets.QApplication([])
+    app.lstart = []
+    if bgui:
+        widget = Widget()
+        widget._app = app
+        widget.show()
+    else:
+        widget = None
+        
+    loop = qasync.QEventLoop(app)
+    asyncio.set_event_loop(loop)
+
+    largs = sys.argv[1:]
+    url = largs[0]
+    outfile = largs[1]
+    jsfile = largs[2] if len(largs) > 2 else None
+    if os.path.exists(url):
+        with open(url, 'rt') as ofd:
+            elts = ofd.readlines()
+            random.shuffle(elts)
+            lelts = elts[:4]
+    else:
+        lelts = [url]
+    for i, elt in enumerate(lelts):
+        # run only starts the url loading
+        r = Render(app, do_print=False, do_save=True)
+        uri = elt.strip()
+        r.run(uri, outfile, jsfile)
+        per = int(float(i)*100.0/2/len(lelts))
+        LOG.debug(f"{r.percent} {app.lstart} {per} {i}")
+        if len(lelts) == 1: break
+        for j in range(1, random.randint(30, 120)):
+            # google throttles too many links at a time
+            if widget:
+                widget.update(str(per))
+            app.processEvents()
+            time.sleep(1)
+    LOG.info(f"queued {len(app.lstart)} urls")
+        
+    # run until app.exec() is finished (Qt window is closed)
+    task = loop.create_task(main(widget, app, len(lelts)))
+    loop.run_forever()
+
+    # cancel remaining tasks and wait for them to complete
+    task.cancel()
+    tasks = asyncio.all_tasks()
+    loop.run_until_complete(asyncio.gather(*tasks))
+
+if __name__ == '__main__':
+    try:
+        from exclude_badExits import vsetup_logging
+        d = int(os.environ.get('DEBUG', 0))
+        if d > 0:
+            vsetup_logging(10, stream=sys.stderr)
+        else:
+            vsetup_logging(20, stream=sys.stderr)
+        vsetup_logging(log_level, logfile='', stream=sys.stderr)
+    except: pass
+    
+    iMain(sys.argv[1:], bgui=False)
+