|
| 1 | +from pathlib import Path |
| 2 | +from redturtle.rsync.interfaces import IRedturtleRsyncAdapter |
| 3 | +from requests.adapters import HTTPAdapter |
| 4 | +from requests.packages.urllib3.util.retry import Retry |
| 5 | +from zope.component import adapter |
| 6 | +from zope.interface import implementer |
| 7 | +from zope.interface import Interface |
| 8 | + |
| 9 | +import json |
| 10 | +import requests |
| 11 | + |
| 12 | + |
| 13 | +class TimeoutHTTPAdapter(HTTPAdapter): |
| 14 | + def __init__(self, *args, **kwargs): |
| 15 | + if "timeout" in kwargs: |
| 16 | + self.timeout = kwargs["timeout"] |
| 17 | + del kwargs["timeout"] |
| 18 | + super(TimeoutHTTPAdapter, self).__init__(*args, **kwargs) |
| 19 | + |
| 20 | + def send(self, request, **kwargs): |
| 21 | + timeout = kwargs.get("timeout") |
| 22 | + if timeout is None: |
| 23 | + kwargs["timeout"] = self.timeout |
| 24 | + return super(TimeoutHTTPAdapter, self).send(request, **kwargs) |
| 25 | + |
| 26 | + |
| 27 | +@implementer(IRedturtleRsyncAdapter) |
| 28 | +@adapter(Interface, Interface) |
| 29 | +class RsyncAdapterBase: |
| 30 | + """ |
| 31 | + This is the base class for all rsync adapters. |
| 32 | + It provides a common interface for all adapters and some default |
| 33 | + implementations of the methods. |
| 34 | + Default methods works with some data in restapi-like format. |
| 35 | + """ |
| 36 | + |
| 37 | + def __init__(self, context, request): |
| 38 | + self.context = context |
| 39 | + self.request = request |
| 40 | + |
| 41 | + def requests_retry_session( |
| 42 | + self, |
| 43 | + retries=3, |
| 44 | + backoff_factor=0.3, |
| 45 | + status_forcelist=(500, 501, 502, 503, 504), |
| 46 | + timeout=5.0, |
| 47 | + session=None, |
| 48 | + ): |
| 49 | + """ |
| 50 | + https://dev.to/ssbozy/python-requests-with-retries-4p03 |
| 51 | + """ |
| 52 | + session = session or requests.Session() |
| 53 | + retry = Retry( |
| 54 | + total=retries, |
| 55 | + read=retries, |
| 56 | + connect=retries, |
| 57 | + backoff_factor=backoff_factor, |
| 58 | + status_forcelist=status_forcelist, |
| 59 | + ) |
| 60 | + # adapter = HTTPAdapter(max_retries=retry) |
| 61 | + http_adapter = TimeoutHTTPAdapter(max_retries=retry, timeout=timeout) |
| 62 | + session.mount("http://", http_adapter) |
| 63 | + session.mount("https://", http_adapter) |
| 64 | + return session |
| 65 | + |
| 66 | + def log_item_title(self, start, options): |
| 67 | + """ |
| 68 | + Return the title of the log item for the rsync command. |
| 69 | + """ |
| 70 | + return f"Report sync {start.strftime('%d-%m-%Y %H:%M:%S')}" |
| 71 | + |
| 72 | + def set_args(self, parser): |
| 73 | + """ |
| 74 | + Set some additional arguments for the rsync command. |
| 75 | +
|
| 76 | + For example: |
| 77 | + parser.add_argument( |
| 78 | + "--import-type", |
| 79 | + choices=["xxx", "yyy", "zzz"], |
| 80 | + help="Import type", |
| 81 | + ) |
| 82 | + """ |
| 83 | + return |
| 84 | + |
| 85 | + def get_data(self, options): |
| 86 | + """ |
| 87 | + Convert the data to be used for the rsync command. |
| 88 | + Return: |
| 89 | + - data: the data to be used for the rsync command |
| 90 | + - error: an error message if there was an error, None otherwise |
| 91 | + """ |
| 92 | + error = None |
| 93 | + data = None |
| 94 | + # first, read source data |
| 95 | + if getattr(options, "source_path", None): |
| 96 | + file_path = Path(options.source_path) |
| 97 | + if file_path.exists() and file_path.is_file(): |
| 98 | + with open(file_path, "r") as f: |
| 99 | + try: |
| 100 | + data = json.load(f) |
| 101 | + except json.JSONDecodeError: |
| 102 | + data = f.read() |
| 103 | + else: |
| 104 | + error = f"Source file not found in: {file_path}" |
| 105 | + return data, error |
| 106 | + elif getattr(options, "source_url", None): |
| 107 | + http = self.requests_retry_session(retries=7, timeout=30.0) |
| 108 | + response = http.get(options.source_url) |
| 109 | + if response.status_code != 200: |
| 110 | + error = f"Error getting data from {options.source_url}: {response.status_code}" |
| 111 | + return data, error |
| 112 | + if "application/json" in response.headers.get("Content-Type", ""): |
| 113 | + try: |
| 114 | + data = response.json() |
| 115 | + except ValueError: |
| 116 | + data = response.content |
| 117 | + else: |
| 118 | + data = response.content |
| 119 | + |
| 120 | + if data: |
| 121 | + data, error = self.convert_source_data(data) |
| 122 | + return data, error |
| 123 | + |
| 124 | + def convert_source_data(self, data): |
| 125 | + """ |
| 126 | + If needed, convert the source data to a format that can be used by the rsync command. |
| 127 | + """ |
| 128 | + return data, None |
| 129 | + |
| 130 | + def find_item_from_row(self, row): |
| 131 | + """ |
| 132 | + Find the item in the context from the given row of data. |
| 133 | + This method should be implemented by subclasses to find the specific type of content item. |
| 134 | + """ |
| 135 | + raise NotImplementedError() |
| 136 | + |
| 137 | + def create_item(self, row, options): |
| 138 | + """ |
| 139 | + Create a new content item from the given row of data. |
| 140 | + This method should be implemented by subclasses to create the specific type of content item. |
| 141 | + """ |
| 142 | + raise NotImplementedError() |
| 143 | + |
| 144 | + def update_item(self, item, row): |
| 145 | + """ |
| 146 | + Update an existing content item from the given row of data. |
| 147 | + This method should be implemented by subclasses to update the specific type of content item. |
| 148 | + """ |
| 149 | + raise NotImplementedError() |
| 150 | + |
| 151 | + def delete_items(self, data, sync_uids): |
| 152 | + """ |
| 153 | + params: |
| 154 | + - data: the data to be used for the rsync command |
| 155 | + - sync_uids: the uids of the items thata has been updated |
| 156 | +
|
| 157 | + Delete items if needed. |
| 158 | + This method should be implemented by subclasses to delete the specific type of content item. |
| 159 | + """ |
| 160 | + return |
0 commit comments