Create some blog posts

.gitignore

@@ -19,3 +19,4 @@ backend/img/plots
 *time_recoder_config.py
 ~$*
 /time_recoder/ValheimDataBase.py
+blog_posts

blog_writer.py

@@ -0,0 +1,117 @@
+import datetime
+import json
+import openai
+import os
+from typing import Optional
+
+from tool_lib.util import EBC
+
+
+class Topic(EBC):
+    def __init__(self, topic_name: str, topic_detailed_description=''):
+        self.topic_name = topic_name
+        self.topic_detailed_description = topic_detailed_description
+
+
+class Template(EBC):
+    def __init__(self, template_name: str, template_content: str, formatting_kwargs: Optional[dict] = None):
+        self.template_name = template_name
+        self.template_content = template_content
+        if formatting_kwargs is None:
+            formatting_kwargs = {}
+        self.formatting_kwargs = formatting_kwargs
+
+    def update_formatting_kwargs(self, **kwargs):
+        self.formatting_kwargs.update(kwargs)
+
+    def format(self, topic: Topic):
+        return self.template_content.format(topic_name=topic.topic_name, topic_detailed_description=topic.topic_detailed_description, **self.formatting_kwargs)
+
+
+class EnDBlogTemplate(Template):
+    def __init__(self,
+                 min_words=400,
+                 max_words=600):
+        template_name = 'end_blog_template'
+        template_content = '''For context: You are writing a blog post for publication on the website of our company EnD UG.
+We specialize in tour planning software and are proud that our product is fast, reliable and easy to use.
+It allows our customers to plan the driving routes of their employees automatically while optimizing the driving time, fuel usage and timeliness.
+It also allows our customers to plan the driving routes of their employees manually, while giving them suggestions on how to improve the route.
+
+Please write a blog post about the topic of "{topic_name}" in a length of {min_words} to {max_words}. {topic_detailed_description}
+
+Make sure that the blog post is written in a way that is easy to understand for visitors to our website, including potential customers, but technically detailed enough to show that the writer has expertise in the topic.
+It should be written in an informative and factual way and not be promotional.
+Do not address the reader directly. Write in third person. In particular, avoid phrases like "At EnD UG we ..." or "As a business manager you ..." or "our [company, tool, software, ...]".
+Include a title and format the output as markdown. In particular, after each sentence, add a newline, and an additional one after paragraphs. 
+Do not include URLs, images or references to other blog posts.
+'''
+        super().__init__(template_name, template_content, {'min_words': min_words, 'max_words': max_words})
+
+
+class TechnicalBlogTemplate(Template):
+    def __init__(self,
+                 min_words=400,
+                 max_words=600):
+        mean_words = (min_words + max_words) / 2
+        template_name = 'technical_blog_template'
+        template_content = '''Please write a blog post about the topic of "{topic_name}" in a length of {min_words} to {max_words}, ideally around {mean_words}. {topic_detailed_description}
+
+Make sure that the blog post is written in a way that is easy to understand, but technically detailed enough to show that the writer has expertise in the topic.
+It should be written in an informative and factual way and not be promotional.
+Do not address the reader directly. Write in third person.
+Include a title and format the output as markdown. In particular, after each sentence, add a newline, and an additional one after paragraphs.
+Structure sections with headings.
+Do not include URLs, images or references to other blog posts.
+'''
+        super().__init__(template_name, template_content, {'min_words': min_words, 'max_words': max_words, 'mean_words': mean_words})
+
+
+class BlogCreator(EBC):
+    def __init__(self, template: Template, topic: Topic):
+        self.template = template
+        self.topic = topic
+
+    def request_template(self):
+        return self.template.template_content
+
+    def write(self):
+        message = [{"role": "user",
+                    "content": self.request()}]
+        response = openai.ChatCompletion.create(
+            model="gpt-3.5-turbo",
+            messages=message,
+            temperature=0.2,
+            max_tokens=1000,
+            frequency_penalty=0.0
+        )
+        return response['choices'][0]['message']['content']
+
+    def request(self):
+        return self.template.format(self.topic)
+
+    def to_disk(self):
+        to_file = os.path.abspath(f'blog_posts/{topic.topic_name}_{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.json')
+        os.makedirs(os.path.dirname(to_file), exist_ok=True)
+        result = {'blogger': self.to_json(),
+                  'dt_created': datetime.datetime.now().timestamp(),
+                  'request': self.request(),
+                  'result': self.write()}
+        with open(to_file, 'w', encoding='utf-8') as f:
+            json.dump(result, f, indent=4)
+        with open(to_file.replace('.json', '.md'), 'w', encoding='utf-8') as f:
+            f.write(result['result'])
+        print(f'Wrote {to_file}')
+
+
+if __name__ == '__main__':
+    topics = [
+        # Topic('Travel Salesman'),
+        # Topic('k-opt Algorithm for the Traveling Salesman Problem'),
+        # Topic('Linear Programming'),
+        # Topic('Mixed Linear Programming'),
+        # Topic('Clustering'),
+        # Topic('OpenAPI and Swagger'),
+    ]
+    for topic in topics:
+        BlogCreator(TechnicalBlogTemplate(), topic).to_disk()

tool_lib/compact_dict_string.py

@@ -0,0 +1,93 @@
+def compact_object_string(o, max_line_length=120, indent=0, max_depth=2 ** 32):
+    if max_depth == 0:
+        return ' ' * indent + '...'
+    if isinstance(o, list):
+        return compact_list_string(o, max_line_length, indent=indent, max_depth=max_depth)
+    elif isinstance(o, tuple):
+        return compact_tuple_string(o, max_line_length, indent=indent, max_depth=max_depth)
+    elif isinstance(o, dict):
+        return compact_dict_string(o, max_line_length, indent=indent, max_depth=max_depth)
+    elif isinstance(o, str):
+        return '"' + o + '"'
+    else:
+        return ' ' * indent + str(o)
+
+
+def compact_list_string(xs: list, max_line_length=120, indent=0, closing=']', opening='[', max_depth=2 ** 32):
+    # try to fit everything in one line with the default str method
+    single_line_result = ' ' * indent + str(xs)
+    if len(single_line_result) <= max_line_length:
+        return single_line_result
+
+    # not extra lines for [ and ]
+    multi_line_result_right = ' ' * indent + opening
+    for x in xs:
+        prefix = ''
+        multi_line_result_right += prefix + f'{compact_object_string(x, max_line_length=max_line_length, indent=indent + 1 + len(prefix), max_depth=max_depth - 1)}'.strip()
+        multi_line_result_right += ',\n' + ' ' * indent
+    multi_line_result_right = multi_line_result_right[:-len(',\n' + ' ' * indent)] + closing
+
+    # extra lines for [ and ]
+    multi_line_result_below = ' ' * indent + opening
+    for x in xs:
+        prefix = ''
+        multi_line_result_below += '\n' + ' ' * (indent + 2)
+        multi_line_result_below += prefix + f'{compact_object_string(x, max_line_length=max_line_length, indent=indent + 2 + len(prefix), max_depth=max_depth - 1)}'.strip()
+    multi_line_result_below += ',\n' + ' ' * indent
+    multi_line_result_below = multi_line_result_below + closing
+
+    if len(multi_line_result_right.splitlines()) < len(multi_line_result_below.splitlines()):
+        return multi_line_result_right
+    else:
+        return multi_line_result_below
+
+
+def compact_tuple_string(xs: tuple, max_line_length=120, indent=0, max_depth=2 ** 32):
+    return compact_list_string(list(xs), max_line_length, indent, closing=')', opening='(')
+
+
+def compact_dict_string(d: dict, max_line_length=120, indent=0, max_depth=2 ** 32):
+    # try to fit everything in one line with the default str method
+    single_line_result = ' ' * indent + str(d).replace("'", '"')
+    if len(single_line_result) <= max_line_length:
+        return single_line_result
+
+    # try to put compact value string next to the key strings
+    multi_line_result_right = ' ' * indent + '{'
+    for k, v in d.items():
+        if isinstance(k, str):
+            prefix = '"' + str(k) + '"' + ': '
+        else:
+            prefix = str(k) + ': '
+        multi_line_result_right += prefix + f'{compact_object_string(v, max_line_length=max_line_length, indent=indent + 1 + len(prefix), max_depth=max_depth - 1)}'.strip()
+        multi_line_result_right += ',\n' + ' ' * indent
+    multi_line_result_right = multi_line_result_right[:-len(',\n' + ' ' * indent)] + '}'
+
+    # try to put compact value string below key strings
+    multi_line_result_below = ' ' * indent + '{'
+    multi_line_result_below += '\n' + ' ' * (indent + 2)
+    for k, v in d.items():
+        prefix = '"' + str(k) + '"' + ': '
+        multi_line_result_below += prefix + f'{compact_object_string(v, max_line_length=max_line_length, indent=indent + 2 + len(prefix), max_depth=max_depth - 1)}'.strip()
+        multi_line_result_below += ',\n' + ' ' * (indent + 2)
+    multi_line_result_below = multi_line_result_below[:-2] + '}'
+
+    if len(multi_line_result_right.splitlines()) < len(multi_line_result_below.splitlines()):
+        return multi_line_result_right
+    else:
+        return multi_line_result_below
+
+
+if __name__ == '__main__':
+    print(compact_dict_string({"body": {"game_state": {"crafting_machines": [{"name": "Basic Crafting Machine 16910", "duration": 10, "type": "CraftingMachine"}], "game_name": "test", "properties": [
+        {"tier": 1, "name": "Luxuriant", "type": "Property"},
+        {"tier": 1, "name": "Edible", "type": "Property"},
+        {"tier": 1, "name": "Appealable", "type": "Property"}], "resources": [
+        {"properties": [{"tier": 1, "name": "Luxuriant", "type": "Property"}], "name": "Wolverine", "type": "Resource"},
+        {"properties": [{"tier": 1, "name": "Edible", "type": "Property"}], "name": "Lm", "type": "Resource"},
+        {"properties": [{"tier": 1, "name": "Appealable", "type": "Property"}], "name": "Skittishness", "type": "Resource"}], "users": [
+        {"resource_inventory": {}, "session_id": "12843d4e-8cfb-4f30-acaa-d4d9de6cf33f", "username": "testUser", "type": "User"}], "resource_discoveries":
+                                                           {"testUser": [{"properties": [{"tier": 1, "name": "Luxuriant", "type": "Property"}], "name": "Wolverine", "type": "Resource"},
+                                                                         {"properties": [{"tier": 1, "name": "Edible", "type": "Property"}], "name": "Lm", "type": "Resource"},
+                                                                         {"properties": [{"tier": 1, "name": "Appealable", "type": "Property"}], "name": "Skittishness", "type": "Resource"}]},
+                                                       "type": "GameState"}}, "http_status_code": 200, "request_token": "774ba52b-31a2-481a-9f12-537495dae993"}, max_line_length=200))

tool_lib/compute_cartesic_distance.py

@@ -0,0 +1,17 @@
+from typing import Tuple, List
+
+import numpy
+
+
+# This file uses list_of_coordinates in a 2dimensional cartesic space.
+# To calculate a symmetric matrix (nxn) with n as length of the list_of_coordinates
+
+def compute_symmetric_matrix_in_cartesic_space(coordinates: List[Tuple[float, float]]):
+    if len(coordinates) == 0:
+        return numpy.zeros((0, 0))
+    c = numpy.array(coordinates)
+    m = numpy.meshgrid([range(len(c))], [range(len(c))])
+    xy1 = c[m[0]]
+    xy2 = c[m[1]]
+    matrix = numpy.linalg.norm(xy1 - xy2, ord=2, axis=2)
+    return matrix

tool_lib/compute_distance.py

@@ -0,0 +1,100 @@
+import math as mt
+
+import numpy as np
+
+
+# Efficient calculating of distance(2-dim) matrix, use of symmetric and 0 in diagonals
+# nested sums, first does not contain the last element, second does not contain the first
+# Implemented Geo_graphic Coordsystem
+# https://www.movable-type.co.uk/scripts/latlong.html formula haversine
+# TODO General dimension
+
+
+def get_angle_in_rad(coordinate):
+    return coordinate * mt.pi / 180
+
+
+def calc_haversine_angle(delta_lat, lat1, lat2, delta_long):
+    return (
+        mt.sin(delta_lat / 2) * mt.sin(delta_lat / 2)
+        + mt.cos(lat1) * mt.cos(lat2) * mt.sin(delta_long / 2) * mt.sin(delta_long / 2)
+    )
+
+
+def calc_haversine_distance_km(radius_earth_in_meter, haversine_angle):
+    radius_km = radius_earth_in_meter / 1000
+    return radius_km * 2 * mt.atan2(mt.sqrt(haversine_angle), mt.sqrt(1 - haversine_angle))
+
+
+def get_symmetric_distances_matrix(list_of_coordinates):
+    radius_earth_in_meter = 6371000  # radius*pi/180
+    matrix = np.zeros([len(list_of_coordinates), len(list_of_coordinates)])
+    for rows in range(0, len(list_of_coordinates) - 1):
+        for columns in range(rows + 1, len(list_of_coordinates)):
+            lat1 = get_angle_in_rad(list_of_coordinates[rows][0])
+            lat2 = get_angle_in_rad(list_of_coordinates[columns][0])
+            lon1 = get_angle_in_rad(list_of_coordinates[rows][1])
+            lon2 = get_angle_in_rad(list_of_coordinates[columns][1])
+
+            distance = haversine_distance(lat1, lon1, lat2, lon2, radius_earth_in_meter)
+            matrix[rows, columns] = distance
+            matrix[columns, rows] = matrix[rows, columns]
+    return matrix
+
+
+def haversine_distance(lat1, lon1, lat2, lon2, radius_earth_in_meter=6371000):
+    """input in radiant, output in km"""
+    for input_value in [lat1, lon1, lat2, lon2]:
+        if abs(input_value) > 2 * mt.pi:
+            raise ValueError(f"Input value is not in radiant. Values were {[lat1, lon1, lat2, lon2]}")
+    delta_lat = lat2 - lat1
+    delta_long = lon1 - lon2
+    haversine_angle = calc_haversine_angle(delta_lat, lat1, lat2, delta_long)
+    distance = calc_haversine_distance_km(radius_earth_in_meter, haversine_angle)
+    return distance
+
+
+def haversine_distance_from_degrees(lat1, lon1, lat2, lon2, radius_earth_in_meter=6371000):
+    return haversine_distance(
+        np.deg2rad(lat1),
+        np.deg2rad(lon1),
+        np.deg2rad(lat2),
+        np.deg2rad(lon2),
+        radius_earth_in_meter
+    )
+
+
+def get_row_of_distances(list_of_coordinates, start_coordinate):
+    '''
+    :param list_of_coordinates: list of coordinate tuple
+    :param start_coordinate: tuple of latitude and longitude of a specific node
+    :return:
+    '''
+    radius_earth_in_meter = 6371000  # radius*pi/180
+    array = np.zeros([len(list_of_coordinates), 1])
+    for rows in range(0, len(list_of_coordinates)):
+        lat1 = get_angle_in_rad(list_of_coordinates[rows][0])
+        lat2 = get_angle_in_rad(start_coordinate[0])
+        lon1 = get_angle_in_rad(list_of_coordinates[rows][1])
+        lon2 = get_angle_in_rad(start_coordinate[1])
+
+        delta_lat = lat2 - lat1
+        delta_long = lon1 - lon2
+
+        haversine_angle = calc_haversine_angle(delta_lat, lat1, lat2, delta_long)
+        distance = calc_haversine_distance_km(radius_earth_in_meter, haversine_angle)
+        array[rows] = distance
+    return array
+
+
+def geograpic_coord_to_cartesic(value):
+    '''
+    :param value: tuple latidude, longitude
+    :return: tuple, x,y
+    '''
+    lat, lon = np.deg2rad(value[0]), np.deg2rad(value[1])
+    radius_earth_in_km = 6371
+    x = radius_earth_in_km * mt.cos(lat) * mt.cos(lon)
+    y = radius_earth_in_km * mt.cos(lat) * mt.sin(lon)
+
+    return (x, y)

tool_lib/custom_data_tables.py

@@ -0,0 +1,62 @@
+import sqlite3
+
+
+def create_custom_datatable(database_path, columns: list or dict, table_name=''):
+    if table_name == '':
+        return
+    # Create a String
+    query_str = ''#
+    if isinstance(columns, list):
+        for name_count in range(0, len(columns)):
+            query_str += columns[name_count] + ' VARCHAR, '
+    else:
+        for key,v in columns.items():
+            query_str += key + ' VARCHAR, '
+
+
+    query_str= 'CREATE TABLE IF NOT EXISTS ' + table_name + '(' \
+               + query_str[:-2] + ')'
+
+    # Connect to database
+    connection = sqlite3.connect(database_path)
+    cursor = connection.cursor()
+    cursor.execute(query_str)
+    connection.commit()
+    connection.close()
+
+def add_row(database_path, columns:dict, table_name =''):
+    if table_name == '':
+        return
+    # Create a String
+    column_names_str = ''
+    values_list = []
+
+    for key,value in columns.items():
+        column_names_str += key + ', '
+        values_list += [str(value)]
+
+    add_questionmarks = '?,' * len(values_list)
+    query_str = 'INSERT OR IGNORE INTO ' + table_name + '(' + column_names_str[:-2]+  ') VALUES(' + add_questionmarks[:-1] + ')'
+
+    # Connect to database
+    connection = sqlite3.connect(database_path)
+    cursor = connection.cursor()
+    cursor.execute(query_str, values_list)
+    connection.commit()
+    connection.close()
+
+def delete_rows_based_on_high_value(database_path,value, column:str, table_name =''):
+    if table_name == '':
+        return
+
+    # Create a String
+    query_str = 'DELETE FROM ' + table_name + ' WHERE ' + 'CAST('+ column+ ' as decimal)' + ' > ' + str(value)
+
+    # Connect to database
+    connection = sqlite3.connect(database_path)
+    cursor = connection.cursor()
+    cursor.execute(query_str)
+    connection.commit()
+    connection.close()
+
+

tool_lib/date_time.py

@@ -0,0 +1,237 @@
+import cachetools
+import datetime
+import math
+from typing import List, Dict, Any, Type, Optional
+
+from lib.util import EBC
+
+
+def get_timestamp(start_day, input_format) -> int:
+    if isinstance(start_day, int):
+        return start_day
+    elif isinstance(start_day, str):
+        return int(datetime.datetime.timestamp(datetime.datetime.strptime(start_day, input_format)))
+    else:
+        return int(datetime.datetime.timestamp(start_day))
+
+
+class Day(EBC):
+    def __init__(self, dt: datetime.date, day_id: int):
+        self.day_id = day_id
+        self.dt = dt
+        self._ymd = self.dt.strftime('%Y-%m-%d')
+
+    @classmethod
+    def field_types(cls) -> Dict[str, Type]:
+        result = super().field_types()
+        result['dt'] = SerializableDateTime
+        return result
+
+    def timestamp(self):
+        dt = self.dt
+        if isinstance(dt, datetime.date):
+            dt = datetime.datetime(dt.year, dt.month, dt.day)
+        return dt.timestamp()
+
+    @staticmethod
+    def from_ts(ts: int, day_id: int):
+        return Day(
+            day_id=day_id,
+            dt=datetime.date.fromtimestamp(ts)
+        )
+
+    def day_idx(self):
+        return self.day_id - 1
+
+    def datetime_from_time_string(self, time_string: str):
+        return self.datetime_from_date_and_time_string(self.dt, time_string)
+
+    @staticmethod
+    def datetime_from_date_and_time_string(date: datetime.date, time_string: str):
+        if len(time_string) == 5:
+            time_string += ':00'
+        return datetime.datetime.strptime(f'{date.strftime("%Y-%m-%d")} {time_string}', '%Y-%m-%d %H:%M:%S')
+
+    def ymd(self) -> str:
+        return self._ymd
+
+    def weekday(self):
+        return self.dt.weekday()
+
+    def weekday_as_german_string(self):
+        return {
+            'Monday': 'Montag',
+            'Tuesday': 'Dienstag',
+            'Wednesday': 'Mittwoch',
+            'Thursday': 'Donnerstag',
+            'Friday': 'Freitag',
+            'Saturday': 'Samstag',
+            'Sunday': 'Sonntag',
+        }[self.dt.strftime('%A')]
+
+    def __eq__(self, other):
+        if not isinstance(other, type(self)):
+            raise ValueError
+        return self.day_id == other.day_id and self._ymd == other._ymd
+
+
+@cachetools.cached(cachetools.LRUCache(maxsize=128))
+def time_interval_as_days(start_day, day_count, inputformat='%Y-%m-%d') -> List[Day]:
+    '''
+    :param start_day: date
+    :param day_count:
+    :return: list of dates as string
+    '''
+    start_day = get_timestamp(start_day, inputformat)
+    results = [
+        Day.from_ts(start_day + 86400 * day_idx, day_id=day_idx + 1)
+        for day_idx in range(day_count)
+    ]
+    return results
+
+
+def weekday_name(index: int, lan='english'):
+    """
+    :param index: 0..6
+    :return: The weekday as name
+    """
+    if lan == 'english':
+        weekdays = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
+    else:
+        weekdays = ['Montag', 'Dienstag', 'Mittwoch', 'Donnerstag', 'Freitag', 'Samstag', 'Sonntag']
+    return weekdays[index]
+
+
+def get_calendarweek_from_datetime(datetime_obj):
+    return datetime_obj.strftime('%V')
+
+
+def timestamp_from_time_of_day_string(time_string):
+    ts = datetime.datetime.timestamp(datetime.datetime.strptime('1970-' + time_string + ' +0000', '%Y-%H:%M %z'))
+    assert 0 <= ts <= 24 * 60 * 60
+    return ts
+
+
+class SerializableDateTime(datetime.datetime, EBC):
+    @classmethod
+    def field_types(cls) -> Dict[str, Type]:
+        return {
+            'year': int,
+            'month': int,
+            'day': int,
+            'hour': int,
+            'minute': int,
+            'second': int,
+            'microsecond': int,
+            'tzinfo': type(None),
+        }
+
+    def to_json(self) -> Dict[str, Any]:
+        return {
+            'type': type(self).__name__,
+            'dt_str': self.strftime('%Y-%m-%d %H:%M:%S.%f'),
+        }
+
+    def __hash__(self):
+        return super().__hash__()
+
+    @staticmethod
+    def from_datetime(dt):
+        return SerializableDateTime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.microsecond)
+
+    @staticmethod
+    def from_date(dt):
+        if isinstance(dt, datetime.datetime):
+            return SerializableDateTime.from_datetime(dt)
+        return SerializableDateTime(dt.year, dt.month, dt.day)
+
+    @staticmethod
+    def from_json(data: Dict[str, Any]):
+        cls: Type[SerializableDateTime] = EBC.SUBCLASSES_BY_NAME[data['type']]
+        if 'dt_str' in data:
+            dt = datetime.datetime.strptime(data['dt_str'], '%Y-%m-%d %H:%M:%S.%f')
+            data['year'] = dt.year
+            data['month'] = dt.month
+            data['day'] = dt.day
+            data['hour'] = dt.hour
+            data['minute'] = dt.minute
+            data['second'] = dt.second
+            data['microsecond'] = dt.microsecond
+        return cls(data['year'], data['month'], data['day'], data['hour'], data['minute'], data['second'], data['microsecond'], data.get('tzinfo', None))
+
+    def __eq__(self, other):
+        if isinstance(other, datetime.date) and not isinstance(other, datetime.datetime):
+            return self.date() == other
+        return (
+                self.year == other.year
+                and self.month == other.month
+                and self.day == other.day
+                and self.hour == other.hour
+                and self.minute == other.minute
+                and self.second == other.second
+                and self.microsecond == other.microsecond
+                and self.tzinfo == other.tzinfo
+        )
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class SerializableTimeDelta(datetime.timedelta, EBC):
+    max: 'SerializableTimeDelta'
+
+    @classmethod
+    def field_types(cls) -> Dict[str, Type]:
+        return {
+            'seconds': float,
+            'days': Optional[int],
+            'microseconds': Optional[int],
+        }
+
+    def to_json(self) -> Dict[str, Any]:
+        return {
+            'type': type(self).__name__,
+            'seconds': self.seconds,
+            'days': self.days,
+            'microseconds': self.microseconds,
+        }
+
+    @classmethod
+    def from_total_seconds(cls, total_seconds) -> 'SerializableTimeDelta':
+        try:
+            return SerializableTimeDelta(seconds=total_seconds)
+        except OverflowError as e:
+            if 'must have magnitude <=' in str(e):
+                return cls.max
+
+    def positive_infinite(self):
+        return self.days == self.max.days and self.seconds == self.max.seconds and self.microseconds == self.max.microseconds
+
+    def total_seconds(self) -> float:
+        if self.positive_infinite():
+            return math.inf
+        return super().total_seconds()
+
+    @classmethod
+    def from_timedelta(cls, timedelta: datetime.timedelta):
+        return cls(timedelta.days, timedelta.seconds, timedelta.microseconds)
+
+    @staticmethod
+    def from_json(data: Dict[str, Any]):
+        cls: Type[SerializableTimeDelta] = EBC.SUBCLASSES_BY_NAME[data['type']]
+        return cls(data.get('days', 0.), data['seconds'], data.get('microseconds', 0.))
+
+    def __eq__(self, other):
+        return self.total_seconds() == other.total_seconds()
+
+
+SerializableTimeDelta.max = SerializableTimeDelta.from_timedelta(datetime.timedelta.max)
+
+
+def minutes_timestamp_from_time_of_day_string(time_of_day_string):
+    timestamp = timestamp_from_time_of_day_string(time_of_day_string)
+    return minutes_timestamp_from_seconds(timestamp)
+
+
+def minutes_timestamp_from_seconds(timestamp):
+    return timestamp / 60

tool_lib/filter_classes.py

@@ -0,0 +1,29 @@
+'''
+This file is used for filter. Each filter contains different Options. And should be used for different format types.
+If you don't find a fitting filter then program it in this file to structure your coding.
+If there is a similar Filter which does not perfectly fit into your code, please extend this Filter instead of creating a new one
+
+Another Note: Try to keep up with this format: so with a lot of "coding Notes", the Class system and the variable_names
+If you have to use non-class filter please describe why and when to use this non-class filter
+'''
+from typing import Union
+
+import datetime
+import numpy as np
+
+class ManualTimeFilter:
+    '''
+    This filter, takes a Key(axe) which maps on Values on a Matrix
+                 takes an Matrix in which Values are filtered
+                 takes a timeconstraint
+
+    Example: Key= Johann, Matrix= Arbeitszeitmatrix , timeconstraint= Monday
+    Result: Filtered Version of this Matrix
+    '''
+    def __init__(self, key: str, matrix: Union[np.array, list], timeconstraint: Union[str, datetime]):
+        self.key= key
+        self.matrix = matrix
+        self.timeconstraint = timeconstraint
+
+    def __call__(self):
+        return self.matrix

tool_lib/find_functions.py

@@ -0,0 +1,12 @@
+# This File Contains Functions which search for a key in an format and return the desired searched value.
+def find(key, value):
+    for k, v in value.items():
+        if k == key:
+            yield v
+        elif isinstance(v, dict):
+            for result in find(key, v):
+                yield result
+        elif isinstance(v, list):
+            for d in v:
+                for result in find(key, d):
+                    yield result

tool_lib/format_transformations.py

@@ -0,0 +1,21 @@
+
+# This file is written to do type transformations. A transformation symbolizes a mapping.
+
+def asciify(s):
+    replaces = [
+        ('ö', 'oe'),
+        ('ä', 'ae'),
+        ('ü', 'ue'),
+        # (' ', ''),
+        # ('/', ''),
+        # ('(', ''),
+        # (')', ''),
+        ('ß', 'ss'),
+    ]
+    for rep in replaces:
+        s = s.replace(*rep)
+    return s
+
+def json_to_dict(data):
+    import json
+    return json.loads(data)

tool_lib/geojson_map.py

@@ -0,0 +1,36 @@
+import json
+from typing import Optional
+
+import cachetools
+
+
+@cachetools.cached(cache=cachetools.LRUCache(maxsize=10000))
+def latitude_of_zip_code_from_geojson(zip_code: str) -> Optional[float]:
+    try:
+        int(zip_code)
+    except ValueError:
+        return None
+    for f in geojson_data['features']:
+        if int(f['properties']['plz']) == int(zip_code):
+            return f['geometry']['coordinates'][1]
+    return None
+
+
+@cachetools.cached(cache=cachetools.LRUCache(maxsize=10000))
+def zip_code_exists(zip_code: str) -> bool:
+    return latitude_of_zip_code_from_geojson(zip_code) is not None
+
+
+@cachetools.cached(cache=cachetools.LRUCache(maxsize=10000))
+def longitude_of_zip_code_from_geojson(zip_code: str) -> Optional[float]:
+    try:
+        int(zip_code)
+    except ValueError:
+        return None
+    for f in geojson_data['features']:
+        if int(f['properties']['plz']) == int(zip_code):
+            return f['geometry']['coordinates'][0]
+    return None
+
+
+geojson_data = json.loads(open('resources/plz-5stellig-centroid.geojson').read())

tool_lib/infinite_timer.py

@@ -0,0 +1,44 @@
+from threading import Timer
+
+from lib.util import EBC
+
+
+class InfiniteTimer(EBC):
+    """
+    A Timer class that does not stop, unless you want it to.
+    https://stackoverflow.com/a/41450617
+    """
+
+    def __init__(self, seconds, target, daemon=True):
+        self._should_continue = False
+        self.is_running = False
+        self.seconds = seconds
+        self.target = target
+        self.thread = None
+        self.daemon = daemon
+
+    def _handle_target(self):
+        self.is_running = True
+        self.target()
+        self.is_running = False
+        self._start_timer()
+
+    def _start_timer(self):
+        if self._should_continue:  # Code could have been running when cancel was called.
+            self.thread = Timer(self.seconds, self._handle_target)
+            self.thread.daemon = self.daemon
+            self.thread.start()
+
+    def start(self):
+        if not self._should_continue and not self.is_running:
+            self._should_continue = True
+            self._start_timer()
+        else:
+            print("Timer already started or running, please wait if you're restarting.")
+
+    def cancel(self):
+        if self.thread is not None:
+            self._should_continue = False  # Just in case thread is running and cancel fails.
+            self.thread.cancel()
+        else:
+            print("Timer never started or failed to initialize.")

tool_lib/main_wrapper.py

@@ -0,0 +1,140 @@
+import functools
+import inspect
+import os
+import time
+
+from lib.memory_control import MemoryLimiter
+from lib.my_logger import logging
+from lib.print_exc_plus import print_exc_plus
+
+try:
+    import winsound as win_sound
+
+
+    def beep(*args, **kwargs):
+        win_sound.Beep(*args, **kwargs)
+except ImportError:
+    win_sound = None
+    beep = lambda x, y: ...
+
+ENABLE_PROFILING = False
+
+
+def start_profiling():
+    try:
+        import yappi
+    except ModuleNotFoundError:
+        return
+    yappi.set_clock_type("wall")
+    print(f'Starting yappi profiler.')
+    yappi.start()
+
+
+def currently_profiling_yappi():
+    try:
+        import yappi
+    except ModuleNotFoundError:
+        return False
+    return len(yappi.get_func_stats()) > 0
+
+
+def profile_wall_time_instead_if_profiling():
+    try:
+        import yappi
+    except ModuleNotFoundError:
+        return
+    currently_profiling = len(yappi.get_func_stats())
+    if currently_profiling and yappi.get_clock_type() != 'wall':
+        yappi.stop()
+        print('Profiling wall time instead of cpu time.')
+        yappi.clear_stats()
+        yappi.set_clock_type("wall")
+        yappi.start()
+
+
+def dump_pstats_if_profiling(relating_to_object):
+    try:
+        import yappi
+    except ModuleNotFoundError:
+        return
+    currently_profiling = len(yappi.get_func_stats())
+    if currently_profiling:
+        try:
+            pstats_file = 'logs/profiling/' + os.path.normpath(inspect.getfile(relating_to_object)).replace(os.path.abspath('.'), '') + '.pstat'
+        except AttributeError:
+            print('WARNING: unable to set pstat file path for profiling.')
+            return
+        os.makedirs(os.path.dirname(pstats_file), exist_ok=True)
+        yappi.get_func_stats()._save_as_PSTAT(pstats_file)
+        print(f'Saved profiling log to {pstats_file}.')
+
+
+class YappiProfiler():
+    def __init__(self, relating_to_object):
+        self.relating_to_object = relating_to_object
+
+    def __enter__(self):
+        start_profiling()
+        profile_wall_time_instead_if_profiling()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        dump_pstats_if_profiling(self.relating_to_object)
+
+
+def list_logger(base_logging_function, store_in_list: list):
+    def print_and_store(*args, **kwargs):
+        base_logging_function(*args, **kwargs)
+        store_in_list.extend(args)
+
+    return print_and_store
+
+
+EMAIL_CRASHES_TO = []
+
+
+def main_wrapper(f):
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+        if ENABLE_PROFILING:
+            start_profiling()
+        from lib.memory_control import MemoryLimiter
+        MemoryLimiter.limit_memory_usage()
+        start = time.perf_counter()
+        # import lib.stack_tracer
+        import __main__
+        # does not help much
+        # monitoring_thread = hanging_threads.start_monitoring(seconds_frozen=180, test_interval=1000)
+        os.makedirs('logs', exist_ok=True)
+        # stack_tracer.trace_start('logs/' + os.path.split(__main__.__file__)[-1] + '.html', interval=5)
+        # faulthandler.enable()
+        profile_wall_time_instead_if_profiling()
+
+        # noinspection PyBroadException
+        try:
+            return f(*args, **kwargs)
+        except KeyboardInterrupt:
+            error_messages = []
+            print_exc_plus(print=list_logger(logging.error, error_messages),
+                           serialize_to='logs/' + os.path.split(__main__.__file__)[-1] + '.dill')
+        except:
+            error_messages = []
+            print_exc_plus(print=list_logger(logging.error, error_messages),
+                           serialize_to='logs/' + os.path.split(__main__.__file__)[-1] + '.dill')
+            for recipient in EMAIL_CRASHES_TO:
+                from jobs.sending_emails import send_mail
+                send_mail.create_simple_mail_via_gmail(body='\n'.join(error_messages), filepath=None, excel_name=None, to_mail=recipient, subject='Crash report')
+        finally:
+            logging.info('Terminated.')
+            total_time = time.perf_counter() - start
+            # faulthandler.disable()
+            # stack_tracer.trace_stop()
+            frequency = 2000
+            duration = 500
+            beep(frequency, duration)
+            if ENABLE_PROFILING:
+                dump_pstats_if_profiling(f)
+            print('Total time', total_time)
+
+    wrapper.func = f
+
+    return wrapper

tool_lib/memory_control.py

@@ -0,0 +1,311 @@
+import datetime
+import gc
+import logging
+import os
+import re
+import subprocess
+from collections import OrderedDict
+from typing import Union
+from time import sleep
+
+import psutil
+
+try:
+    from config import USE_MEMORY_CONTROL
+except ImportError:
+    USE_MEMORY_CONTROL = False
+try:
+    from config import GPU_MEMORY_USAGE
+except ImportError:
+    GPU_MEMORY_USAGE: Union[float, str] = 'growth'
+
+
+class NVLog(dict):
+    __indent_re__ = re.compile('^ *')
+    __version_re__ = re.compile(r'v([0-9.]+)$')
+
+    def __init__(self):
+        super().__init__()
+
+        lines = run_cmd(['nvidia-smi', '-q'])
+        lines = lines.splitlines()
+        while '' in lines:
+            lines.remove('')
+
+        path = [self]
+        self['version'] = self.__version__()
+        for line in lines[1:]:
+            indent = NVLog.__get_indent__(line)
+            line = NVLog.__parse_key_value_pair__(line)
+            while indent < len(path) * 4 - 4:
+                path.pop()
+            cursor = path[-1]
+            if len(line) == 1:
+                if line[0] == 'Processes':
+                    cursor[line[0]] = []
+                else:
+                    cursor[line[0]] = {}
+                cursor = cursor[line[0]]
+                path.append(cursor)
+            elif len(line) == 2:
+                if line[0] in ['GPU instance ID', 'Compute instance ID']:
+                    continue
+                if line[0] == 'Process ID':
+                    cursor.append({})
+                    cursor = cursor[-1]
+                    path.append(cursor)
+                cursor[line[0]] = line[1]
+
+        self['Attached GPUs'] = OrderedDict()
+        keys = list(self.keys())
+        for i in keys:
+            if i.startswith('GPU '):
+                self['Attached GPUs'][i] = self[i]
+                del self[i]
+
+    @staticmethod
+    def __get_indent__(line):
+        return len(NVLog.__indent_re__.match(line).group())
+
+    @staticmethod
+    def __parse_key_value_pair__(line):
+        result = line.split(' : ')
+        result[0] = result[0].strip()
+        if len(result) > 1:
+            try:
+                result[1] = int(result[1])
+            except:
+                pass
+            if result[1] in ['N/A', 'None']:
+                result[1] = None
+            if result[1] in ['Disabled', 'No']:
+                result[1] = False
+        return result
+
+    def __get_processes__(self):
+        processes = []
+        for i, gpu in enumerate(self['Attached GPUs']):
+            gpu = self['Attached GPUs'][gpu]
+            if gpu['Processes']:
+                for j in gpu['Processes']:
+                    processes.append((i, j))
+        return processes
+
+    @staticmethod
+    def __version__():
+        lines = run_cmd(['nvidia-smi', '-h'])
+        lines = lines.splitlines()
+        result = NVLog.__version_re__.search(lines[0]).group(1)
+        return result
+
+    def gpu_table(self):
+        output = []
+        output.append(self['Timestamp'])
+        output.append('+-----------------------------------------------------------------------------+')
+        values = []
+        values.append(self['version'])
+        values.append(self['Driver Version'])
+        if 'CUDA Version' in self:
+            values.append(self['CUDA Version'])
+        else:
+            values.append('N/A')
+        output.append('| NVIDIA-SMI %s       Driver Version: %s       CUDA Version: %-5s    |' % tuple(values))
+        output.append('|-------------------------------+----------------------+----------------------+')
+        output.append('| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |')
+        output.append('| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |')
+        output.append('|===============================+======================+======================|')
+        for i, gpu in enumerate(self['Attached GPUs']):
+            gpu = self['Attached GPUs'][gpu]
+            values = []
+            values.append(i)
+            values.append(gpu['Product Name'])
+            values.append('On' if gpu['Persistence Mode'] else 'Off')
+            values.append(gpu['PCI']['Bus Id'])
+            values.append('On' if gpu['Display Active'] else 'Off')
+            output.append('|   %d  %-19s %3s  | %s %3s |                  N/A |' % tuple(values))
+            values = []
+            values.append(gpu['Fan Speed'].replace(' ', ''))
+            values.append(gpu['Temperature']['GPU Current Temp'].replace(' ', ''))
+            values.append(gpu['Performance State'])
+            values.append(int(float(gpu['Power Readings']['Power Draw'][:-2])))
+            values.append(int(float(gpu['Power Readings']['Power Limit'][:-2])))
+            values.append(gpu['FB Memory Usage']['Used'].replace(' ', ''))
+            values.append(gpu['FB Memory Usage']['Total'].replace(' ', ''))
+            values.append(gpu['Utilization']['Gpu'].replace(' ', ''))
+            values.append(gpu['Compute Mode'])
+            output.append('| %3s   %3s    %s   %3dW / %3dW |  %8s / %8s |    %4s     %8s |' % tuple(values))
+            output.append('+-----------------------------------------------------------------------------+')
+        return '\n'.join(output)
+
+    def processes_table(self):
+        output = []
+        output.append('+-----------------------------------------------------------------------------+')
+        output.append('| Processes:                                                       GPU Memory |')
+        output.append('|  GPU       PID   Type   Process name                             Usage      |')
+        output.append('|=============================================================================|')
+        processes = self.__get_processes__()
+        if len(processes) == 0:
+            output.append('|  No running processes found                                                 |')
+        for i, process in processes:
+            values = []
+            values.append(i)
+            values.append(process['Process ID'])
+            values.append(process['Type'])
+            if len(process['Name']) > 42:
+                values.append(process['Name'][:39] + '...')
+            else:
+                values.append(process['Name'])
+            values.append(process['Used GPU Memory'].replace(' ', ''))
+            output.append('|   %2d     %5d %6s   %-42s %8s |' % tuple(values))
+        output.append('+-----------------------------------------------------------------------------+')
+        return '\n'.join(output)
+
+    def as_table(self):
+        output = []
+        output.append(self.gpu_table())
+        output.append('')
+        output.append(self.processes_table())
+        return '\n'.join(output)
+
+
+class NVLogPlus(NVLog):
+
+    def processes_table(self):
+        output = ['+-----------------------------------------------------------------------------+',
+                  '| Processes:                                                       GPU Memory |',
+                  '|  GPU       PID   User   Process name                             Usage      |',
+                  '|=============================================================================|']
+        processes = self.__get_processes__()
+        if len(processes) == 0:
+            output.append('|  No running processes found                                                 |')
+        for i, process in processes:
+            values = []
+            values.append(i)
+            values.append(process['Process ID'])
+            p = psutil.Process(process['Process ID'])
+            with p.oneshot():
+                values.append(p.username()[:8].center(8))
+                command = p.cmdline()
+                command[0] = os.path.basename(command[0])
+                command = ' '.join(command)
+                if len(command) > 42:
+                    values.append(command[:39] + '...')
+                else:
+                    values.append(command)
+            values.append(process['Used GPU Memory'].replace(' ', ''))
+            output.append('|   %2d     %5d %8s %-42s %8s |' % tuple(values))
+        output.append('+-----------------------------------------------------------------------------+')
+        return '\n'.join(output)
+
+
+def run_cmd(cmd):
+    return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.decode('utf-8')
+
+
+class MemoryLimiter:
+    limited = False
+
+    @classmethod
+    def limit_memory_usage(cls, verbose=1):
+        if not USE_MEMORY_CONTROL:
+            print('memory_control module disabled.')
+            return
+        if cls.limited:
+            if verbose:
+                print('Already limited memory usage. Skipping...')
+            return
+        use_gpu_idx = None
+        printed_compute_mode = False
+        while True:  # until a GPU is free
+            try:
+                data = NVLog()
+            except FileNotFoundError:
+                print('WARNING: nvidia-smi is not available')
+                break
+            if 'use_specific_gpu' in os.environ:
+                print(f'use_specific_gpu = {os.environ["use_specific_gpu"]}')
+                use_gpu_idx = int(os.environ['use_specific_gpu'])
+            else:
+                for idx, gpu_data in reversed(list(enumerate(data['Attached GPUs'].values()))):
+                    any_processes = (
+                            gpu_data['Processes'] is not None
+                            or gpu_data['Utilization']['Memory'] != '0 %'
+                            or gpu_data['FB Memory Usage']['Used'] != '0 MiB'
+                    )
+                    compute_mode = gpu_data['Compute Mode']
+                    if not printed_compute_mode:
+                        print('GPU Compute Mode:', compute_mode)
+                        printed_compute_mode = True
+                    if compute_mode in ['Exclusive_Process', 'Exclusive_Thread'] or os.environ.get('use_empty_gpu'):
+                        if not any_processes:
+                            use_gpu_idx = idx
+                            break
+                    elif compute_mode == 'Default':
+                        if GPU_MEMORY_USAGE != 'growth':
+                            free_memory = int(re.search(r'(\d+) MiB', gpu_data['FB Memory Usage']['Free']).group(1))
+                            if free_memory > 2.5 * GPU_MEMORY_USAGE:
+                                use_gpu_idx = idx
+                                break
+                        else:
+                            use_gpu_idx = idx
+                            break
+                    elif compute_mode == 'Prohibited':
+                        continue
+                    else:
+                        raise NotImplementedError(f'Unknown compute mode: {compute_mode}.')
+                else:
+                    print(datetime.datetime.now().strftime("%H:%M") + ': All GPUs are currently in use.')
+                    sleep(300)
+                    continue
+            os.environ["CUDA_VISIBLE_DEVICES"] = str(use_gpu_idx)
+            print('Using GPU', f'{use_gpu_idx}:', list(data['Attached GPUs'].values())[use_gpu_idx]['Product Name'])
+            break
+        import tensorflow as tf
+
+        # limit GPU memory usage
+        # gpu_memory_limit = 3.5 * 1024
+        for gpu in tf.config.experimental.list_physical_devices('GPU'):
+            if GPU_MEMORY_USAGE == 'growth':
+                tf.config.experimental.set_memory_growth(gpu, True)
+            else:
+                tf.config.experimental.set_virtual_device_configuration(gpu, [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=GPU_MEMORY_USAGE)])
+
+        # tf.config.experimental.set_virtual_device_configuration(gpus[use_gpu_idx], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=2)])
+
+        # limit RAM usage
+        try:
+            from lib.memory_limit_windows import create_job, limit_memory, assign_job
+            import psutil
+
+            ram_limit = psutil.virtual_memory().total * 2 // 3
+            print('Limiting RAM usage to {0:,} Bytes.'.format(ram_limit))
+
+            assign_job(create_job())
+            limit_memory(ram_limit)
+        except ModuleNotFoundError:
+            try:
+                from lib.memory_limit_linux import limit_memory, get_memory
+
+                ram_limit = get_memory() * 2 // 3
+                print('Limiting RAM usage to {0:,} Bytes.'.format(ram_limit))
+
+                limit_memory(ram_limit)
+            except ModuleNotFoundError:
+                print('WARNING: Setting memory limit failed. '
+                      'This can happen if you are not on Windows nor on Linux or if you have forgot to install some dependencies.')
+
+        cls.limited = True
+
+
+MemoryLimiter.limit_memory_usage()
+
+
+def tf_memory_leak_cleanup():
+    import tensorflow
+    for obj in gc.get_objects():
+        if isinstance(obj, tensorflow.Graph):
+            if hasattr(obj, '_py_funcs_used_in_graph'):
+                del obj._py_funcs_used_in_graph[:]
+        if isinstance(obj, tensorflow.keras.utils.GeneratorEnqueuer):
+            obj.stop()
+    gc.collect()

tool_lib/my_logger.py

@@ -0,0 +1,58 @@
+import os
+import logging
+import sys
+
+import __main__
+from typing import List, Dict
+
+FORMAT = "%(asctime)-14s %(levelname)-8s %(module)s:%(lineno)s %(message)s"
+
+
+class ListLogger(logging.Handler):  # Inherit from logging.Handler
+    def __init__(self):
+        logging.Handler.__init__(self)
+        self.running = False
+        self.log_lists: Dict[str, List[str]] = {}
+
+    def start(self):
+        self.running = True
+        self.log_lists.clear()
+
+    def stop(self):
+        self.running = False
+        self.log_lists.clear()
+
+    def emit(self, record):
+        if self.running:
+            level_name = record.levelname
+            if hasattr(record, 'end_user') and record.end_user:
+                level_name = 'END_USER_' + level_name
+            self.log_lists.setdefault(level_name, []).append(record.msg)
+
+
+def end_user_warning(msg):
+    logging.warning(msg, extra={'end_user': True})
+
+
+def end_user_info(msg):
+    logging.info(msg, extra={'end_user': True})
+
+
+list_handler = ListLogger()
+
+logging = logging
+try:
+    logfile = 'logs/' + os.path.normpath(__main__.__file__).replace(os.path.abspath('.'), '') + '.log'
+except AttributeError:
+    print('WARNING: unable to set log file path.')
+else:
+    try:
+        os.makedirs(os.path.dirname(logfile), exist_ok=True)
+        logging.basicConfig(filename=logfile, filemode="a+", format=FORMAT)
+    except OSError:
+        print('WARNING: unable to set log file path.')
+stdout_logger = logging.StreamHandler(sys.stdout)
+stdout_logger.setFormatter(logging.Formatter(FORMAT))
+logging.getLogger().addHandler(stdout_logger)
+logging.getLogger().addHandler(list_handler)
+logging.getLogger().setLevel(logging.INFO)

tool_lib/parameter_search.py

@@ -0,0 +1,1545 @@
+"""
+made by Eren Yilmaz
+"""
+import functools
+import itertools
+from copy import deepcopy
+
+from goto import with_goto
+
+from lib import util
+from lib.tuned_cache import TunedMemory
+from lib.progress_bar import ProgressBar
+import pickle
+
+from matplotlib.axes import Axes
+import os
+import random
+import sqlite3
+from datetime import datetime
+from math import inf, nan, ceil, sqrt
+from timeit import default_timer
+from typing import Dict, Any, List, Callable, Optional, Tuple, Iterable, Union
+
+import numpy
+import pandas
+import scipy.stats
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.dates
+
+from lib.util import my_tabulate, round_to_digits, print_progress_bar, heatmap_from_points, LogicError, shorten_name
+
+score_cache = TunedMemory(location='.cache/scores', verbose=0)
+sns.set(style="whitegrid", font_scale=1.5)
+
+# set up db for results
+connection: sqlite3.Connection = sqlite3.connect('random_search_results.db')
+connection.cursor().execute('PRAGMA foreign_keys = 1')
+connection.cursor().execute('PRAGMA journal_mode = WAL')
+connection.cursor().execute('PRAGMA synchronous = NORMAL')
+
+Parameters = Dict[str, Any]
+MetricValue = float
+Metrics = Dict[str, MetricValue]
+History = List[MetricValue]
+
+suppress_intermediate_beeps = False
+
+label = goto = object()
+
+
+class Prediction:
+    def __init__(self, dataset: str, y_true, y_pred, name: str):
+        self.y_pred = y_pred
+        self.y_true = y_true
+        self.name = name
+        if not isinstance(name, str):
+            self.name = str(name)
+        else:
+            self.name = name
+        if not isinstance(dataset, str):
+            raise TypeError
+        self.dataset = dataset
+
+    def __str__(self):
+        return str(self.__dict__)
+
+    def __repr__(self):
+        return self.__class__.__name__ + repr({k: v for k, v in self.__dict__.items() if k != 'predictions'})
+
+
+class EvaluationResult:
+    def __init__(self,
+                 results: Dict[str, MetricValue],
+                 parameters: Parameters = None,
+                 predictions: Optional[List[Prediction]] = None):
+        self.predictions = predictions
+        if self.predictions is not None:
+            self.predictions = self.predictions.copy()
+        else:
+            self.predictions = []
+        if parameters is None:
+            self.parameters = parameters
+        else:
+            self.parameters = parameters.copy()
+        self.results = results.copy()
+
+    def __iter__(self):
+        yield self
+
+    def __eq__(self, other):
+        return (isinstance(other, EvaluationResult)
+                and self.parameters == other.parameters
+                and self.predictions == other.predictions
+                and self.results == other.results)
+
+    def __str__(self):
+        return '{0}{1}'.format(self.__class__.__name__,
+                               {k: getattr(self, k) for k in ['results', 'parameters', 'predictions']})
+
+    def __repr__(self):
+        return self.__class__.__name__ + repr(self.__dict__)
+
+
+assert list(EvaluationResult({}, {})) == list([EvaluationResult({}, {})])
+
+EvaluationFunction = Callable[[Parameters], Union[List[EvaluationResult], EvaluationResult, List[float], float]]
+
+
+class Parameter:
+    def __init__(self, name: str, initial_value, larger_value, smaller_value, first_try_increase=False):
+        self.name = name
+        self.initial_value = initial_value
+        self.larger_value = larger_value
+        self.smaller_value = smaller_value
+        self.first_try_increase = first_try_increase
+
+    def __repr__(self):
+        return self.__class__.__name__ + repr(self.__dict__)
+
+    def copy(self, new_name=None):
+        result: Parameter = deepcopy(self)
+        if new_name is not None:
+            result.name = new_name
+        return result
+
+
+class BoundedParameter(Parameter):
+    def __init__(self, name, initial_value, larger_value, smaller_value, minimum=-inf, maximum=inf,
+                 first_try_increase=False):
+        self.minimum = minimum
+        self.maximum = maximum
+
+        super().__init__(name,
+                         initial_value,
+                         lambda x: self._bounded(larger_value(x)),
+                         lambda x: self._bounded(smaller_value(x)),
+                         first_try_increase=first_try_increase)
+
+        if self.initial_value < self.minimum:
+            raise ValueError('Initial value is lower than minimum value.')
+        if self.initial_value > self.maximum:
+            raise ValueError('Initial value is larger than maximum value.')
+
+    def _bounded(self, y):
+        y = max(self.minimum, y)
+        y = min(self.maximum, y)
+        return y
+
+
+class ConstantParameter(Parameter):
+    def __init__(self, name, value):
+        super().__init__(name,
+                         value,
+                         lambda x: value,
+                         lambda x: value)
+
+
+class BinaryParameter(Parameter):
+    def __init__(self, name, value1, value2):
+        super().__init__(name,
+                         value1,
+                         lambda x: value2 if x == value1 else value1,
+                         lambda x: value2 if x == value1 else value1)
+
+
+class BooleanParameter(Parameter):
+    def __init__(self, name, initial_value: bool):
+        super().__init__(name,
+                         bool(initial_value),
+                         lambda x: not x,
+                         lambda x: not x)
+
+
+class TernaryParameter(Parameter):
+    def __init__(self, name, value1, value2, value3):
+        self.smaller = {value1: value3, value2: value1, value3: value2}
+        self.larger = {value1: value2, value2: value3, value3: value1}
+        super().__init__(name,
+                         value1,
+                         lambda x: self.smaller[x],
+                         lambda x: self.larger[x])
+
+
+class ListParameter(Parameter):
+    def __init__(self, name, initial_value, possible_values: List, first_try_increase=False, circle=False):
+        self.possible_values = possible_values.copy()
+        if initial_value not in self.possible_values:
+            raise ValueError()
+        if len(set(self.possible_values)) != len(self.possible_values):
+            print('WARNING: It seems that there are duplicates in the list of possible values for {0}'.format(name))
+        length = len(self.possible_values)
+        if circle:
+            smaller = lambda x: self.possible_values[(self.possible_values.index(x) + 1) % length]
+            larger = lambda x: self.possible_values[(self.possible_values.index(x) - 1) % length]
+        else:
+            smaller = lambda x: self.possible_values[min(self.possible_values.index(x) + 1, length - 1)]
+            larger = lambda x: self.possible_values[max(self.possible_values.index(x) - 1, 0)]
+
+        super().__init__(name,
+                         initial_value,
+                         smaller,
+                         larger,
+                         first_try_increase=first_try_increase)
+
+
+class ExponentialParameter(BoundedParameter):
+    def __init__(self, name, initial_value, base, minimum=-inf, maximum=inf, first_try_increase=False):
+        super().__init__(name,
+                         initial_value,
+                         lambda x: float(x * base),
+                         lambda x: float(x / base),
+                         minimum,
+                         maximum,
+                         first_try_increase=first_try_increase)
+        self.plot_scale = 'log'
+
+
+class ExponentialIntegerParameter(BoundedParameter):
+    def __init__(self, name, initial_value, base, minimum=-inf, maximum=inf, first_try_increase=False):
+        if minimum != -inf:
+            minimum = round(minimum)
+
+        if maximum != inf:
+            maximum = round(maximum)
+
+        super().__init__(name,
+                         round(initial_value),
+                         lambda x: round(x * base),
+                         lambda x: round(x / base),
+                         minimum,
+                         maximum,
+                         first_try_increase=first_try_increase)
+        self.plot_scale = 'log'
+
+
+class LinearParameter(BoundedParameter):
+    def __init__(self, name, initial_value, summand, minimum=-inf, maximum=inf, first_try_increase=False):
+        super().__init__(name,
+                         initial_value,
+                         lambda x: float(x + summand),
+                         lambda x: float(x - summand),
+                         minimum,
+                         maximum,
+                         first_try_increase=first_try_increase)
+
+
+class LinearIntegerParameter(BoundedParameter):
+    def __init__(self, name, initial_value, summand, minimum=-inf, maximum=inf, first_try_increase=False):
+        super().__init__(name,
+                         initial_value,
+                         lambda x: x + summand,
+                         lambda x: x - summand,
+                         minimum,
+                         maximum,
+                         first_try_increase=first_try_increase)
+
+
+class InvalidParametersError(Exception):
+    def __init__(self, parameters=None):
+        self.parameters = parameters
+
+
+class BadParametersError(InvalidParametersError):
+    pass
+
+
+class InvalidReturnError(Exception):
+    pass
+
+
+class EmptyTableError(Exception):
+    pass
+
+
+EXAMPLE_PARAMS = [
+    ExponentialParameter('learn_rate', 0.001, 10),
+    ExponentialIntegerParameter('hidden_layer_size', 512, 2, minimum=1),
+    LinearIntegerParameter('hidden_layer_count', 3, 1, minimum=0),
+    ExponentialIntegerParameter('epochs', 100, 5, minimum=1),
+    LinearParameter('dropout_rate', 0.5, 0.2, minimum=0, maximum=1),
+]
+
+
+def mean_confidence_interval_size(data, confidence=0.95, force_v: Optional[int] = None,
+                                  force_sem: Optional[float] = None):
+    if len(data) == 0:
+        return nan
+    if force_sem is None:
+        if len(data) == 1:
+            return inf
+        sem = scipy.stats.sem(data)
+    else:
+        sem = force_sem
+    if sem == 0:
+        return 0
+    if force_v is None:
+        v = len(data) - 1
+    else:
+        v = force_v
+    return numpy.mean(data) - scipy.stats.t.interval(confidence,
+                                                     df=v,
+                                                     loc=numpy.mean(data),
+                                                     scale=sem)[0]
+
+
+def try_parameters(experiment_name: str,
+                   evaluate: EvaluationFunction,
+                   params: Dict[str, any],
+                   optimize: Optional[str] = None,
+                   larger_result_is_better: bool = None, ):
+    print('running experiment...')
+    params = params.copy()
+
+    if larger_result_is_better is None and optimize is not None:
+        raise NotImplementedError(
+            'Don\'t know how to optimize {0}. Did you specify `larger_result_is_better`?'.format(optimize))
+    assert larger_result_is_better is not None or optimize is None
+
+    worst_score = -inf if larger_result_is_better else inf
+
+    cursor = connection.cursor()
+    start = default_timer()
+    try:
+        result = evaluate(params)
+        if not isinstance(result, Iterable):
+            result = [result]
+        evaluation_results: List[EvaluationResult] = list(result)
+    except InvalidParametersError as e:
+        if optimize is not None:
+            bad_results: Dict[str, float] = {
+                optimize: worst_score
+            }
+        else:
+            bad_results = {}
+        if e.parameters is None:
+            evaluation_results = [EvaluationResult(
+                parameters=params,
+                results=bad_results
+            )]
+        else:
+            evaluation_results = [EvaluationResult(
+                parameters=e.parameters,
+                results=bad_results
+            )]
+    finally:
+        duration = default_timer() - start
+
+    for idx in range(len(evaluation_results)):
+        if isinstance(evaluation_results[idx], float):
+            evaluation_results[idx] = EvaluationResult(parameters=params,
+                                                       results={optimize: evaluation_results[idx]})
+
+    p_count = 0
+    for evaluation_result in evaluation_results:
+        if evaluation_result.parameters is None:
+            evaluation_result.parameters = params
+        metric_names = sorted(evaluation_result.results.keys())
+        param_names = list(sorted(evaluation_result.parameters.keys()))
+        for metric_name in metric_names:
+            add_metric_column(experiment_name, metric_name, verbose=1)
+        for param_name in param_names:
+            add_parameter_column(experiment_name, param_name, evaluation_result.parameters[param_name], verbose=1)
+
+        if not set(param_names).isdisjoint(metric_names):
+            raise RuntimeError('Metrics and parameter names should be disjoint')
+
+        if optimize is not None and numpy.isnan(evaluation_result.results[optimize]):
+            evaluation_result.results[optimize] = worst_score
+
+        metric_values = [evaluation_result.results[metric_name] for metric_name in metric_names]
+        param_names_comma_separated = ','.join('"' + param_name + '"' for param_name in param_names)
+        metric_names_comma_separated = ','.join('"' + metric_name + '"' for metric_name in metric_names)
+        insert_question_marks = ','.join('?' for _ in range(len(param_names) + len(metric_names)))
+
+        cursor.execute('''
+                INSERT INTO {0} ({1}) VALUES ({2})
+                '''.format(experiment_name,
+                           param_names_comma_separated + ',' + metric_names_comma_separated,
+                           insert_question_marks), (*[evaluation_result.parameters[name] for name in param_names],
+                                                    *metric_values))
+        result_id = cursor.lastrowid
+        assert cursor.execute(f'SELECT COUNT(*) FROM {experiment_name}_predictions WHERE result_id = ? LIMIT 1',
+                              (result_id,)).fetchone()[0] == 0
+        p_count += len(evaluation_result.predictions)
+        dataset_names = [(prediction.dataset, prediction.name) for prediction in evaluation_result.predictions]
+        if len(set(dataset_names)) != len(dataset_names):
+            print('\n'.join(sorted(dsn
+                                   for idx, dsn in dataset_names
+                                   if dsn in dataset_names[idx:])))
+            raise InvalidReturnError(
+                'Every combination of name and dataset in a single evaluation result must be unique.'
+                'There should be a list of duplicates printed above where the number of occurrences'
+                'of an element in the list is the actual number of occurrences minus 1 '
+                '(so only duplicates are listed).')
+        # noinspection SqlResolve
+        cursor.executemany('''
+            INSERT INTO {0}_predictions (dataset, y_true, y_pred, result_id, name) 
+            VALUES (?, ?, ?, ?, ?)
+            '''.format(experiment_name), [(prediction.dataset,
+                                           pickle.dumps(prediction.y_true),
+                                           pickle.dumps(prediction.y_pred),
+                                           result_id,
+                                           prediction.name) for prediction in evaluation_result.predictions])
+
+    connection.commit()
+    print('saved', len(evaluation_results), 'results and', p_count, 'predictions to db')
+    if not suppress_intermediate_beeps:
+        util.beep(1000, 500)
+
+    if optimize is not None:
+        scores = [r.results[optimize] for r in evaluation_results]
+        if larger_result_is_better:
+            best_score = max(scores)
+        else:
+            best_score = min(scores)
+        print('  finished in', duration, 'seconds, best loss/score:', best_score)
+
+    for r in evaluation_results:
+        if list(sorted(r.results.keys())) != list(sorted(metric_names)):
+            raise InvalidReturnError("""
+            Wrong metric names were returned by `evaluate`:
+            Expected metric_names={0}
+            but was {1}.
+            The result was saved to database anyways, possibly with missing values.
+            """.format(list(sorted(metric_names)),
+                       list(sorted(r.results.keys()))))
+
+    return evaluation_results
+
+
+def config_dict_from_param_list(params: List[Parameter]):
+    return {
+        p.name: p.initial_value
+        for p in params
+    }
+
+
+def evaluate_with_initial_params(experiment_name: str,
+                                 params: List[Parameter],
+                                 evaluate: EvaluationFunction,
+                                 optimize: str,
+                                 larger_result_is_better: bool,
+                                 metric_names=None,
+                                 num_experiments=1, ):
+    random_parameter_search(experiment_name=experiment_name,
+                            params=params,
+                            evaluate=evaluate,
+                            optimize=optimize,
+                            larger_result_is_better=larger_result_is_better,
+                            mutation_probability=1.,
+                            no_mutations_probability=0.,
+                            max_num_experiments=num_experiments,
+                            metric_names=metric_names,
+                            initial_experiments=num_experiments,
+                            experiment_count='db_tries_initial', )
+
+
+def random_parameter_search(experiment_name: str,
+                            params: List[Parameter],
+                            evaluate: EvaluationFunction,
+                            optimize: str,
+                            larger_result_is_better: bool,
+                            mutation_probability: float = None,
+                            no_mutations_probability: float = None,
+                            allow_multiple_mutations=False,
+                            max_num_experiments=inf,
+                            metric_names=None,
+                            initial_experiments=1,
+                            runs_per_configuration=inf,
+                            initial_runs=1,
+                            ignore_configuration_condition='0',
+                            experiment_count='tries', ):
+    print('experiment name:', experiment_name)
+    if metric_names is None:
+        metric_names = [optimize]
+    if optimize not in metric_names:
+        raise ValueError('trying to optimize {0} but only metrics available are {1}'.format(optimize, metric_names))
+    params = sorted(params, key=lambda p: p.name)
+    validate_parameter_set(params)
+
+    param_names = [param.name for param in params]
+
+    cursor = connection.cursor()
+    create_experiment_tables_if_not_exists(experiment_name, params, metric_names)
+
+    def max_tries_reached(ps):
+        return len(result_ids_for_parameters(experiment_name, ps)) >= runs_per_configuration
+
+    def min_tries_reached(ps):
+        return len(result_ids_for_parameters(experiment_name, ps)) >= initial_runs
+
+    def try_(ps) -> bool:
+        tried = False
+        if not max_tries_reached(ps):
+            try_parameters(experiment_name=experiment_name,
+                           evaluate=evaluate,
+                           params=ps,
+                           optimize=optimize,
+                           larger_result_is_better=larger_result_is_better, )
+            tried = True
+        else:
+            print('Skipping because maximum number of tries is already reached.')
+        while not min_tries_reached(ps):
+            print('Repeating because minimum number of tries is not reached.')
+            try_parameters(experiment_name=experiment_name,
+                           evaluate=evaluate,
+                           params=ps,
+                           optimize=optimize,
+                           larger_result_is_better=larger_result_is_better, )
+            tried = True
+        return tried
+
+    if mutation_probability is None:
+        mutation_probability = 1 / (len(params) + 1)
+    if no_mutations_probability is None:
+        no_mutations_probability = (1 - 1 / len(params)) / 4
+    initial_params = {param.name: param.initial_value for param in params}
+    print('initial parameters:', initial_params)
+
+    def skip():
+        best_scores, best_mean, best_std, best_conf = get_results_for_params(optimize, experiment_name,
+                                                                             best_params, 0.99)
+        try_scores, try_mean, try_std, try_conf = get_results_for_params(optimize, experiment_name,
+                                                                         try_params, 0.99)
+        if larger_result_is_better:
+            if best_mean - best_conf > try_mean + try_conf:
+                return True
+        else:
+            if best_mean + best_conf < try_mean - try_conf:
+                return True
+        return False
+
+    # get best params
+    initial_params = {param.name: param.initial_value for param in params}
+    any_results = cursor.execute('SELECT EXISTS (SELECT * FROM {0} WHERE NOT ({1}))'.format(experiment_name,
+                                                                                            ignore_configuration_condition)).fetchone()[
+        0]
+    if any_results:
+        best_params = get_best_params(experiment_name,
+                                      larger_result_is_better,
+                                      optimize,
+                                      param_names,
+                                      additional_condition=f'NOT ({ignore_configuration_condition})')
+    else:
+        best_params = initial_params
+    try_params = best_params.copy()
+
+    def results_for_params(ps):
+        return get_results_for_params(
+            metric=optimize,
+            experiment_name=experiment_name,
+            parameters=ps
+        )
+
+    if experiment_count == 'tries':
+        num_experiments = 0
+    elif experiment_count == 'results':
+        num_experiments = 0
+    elif experiment_count == 'db_total':
+        num_experiments = cursor.execute('SELECT COUNT(*) FROM {0} WHERE NOT ({1})'.format(experiment_name,
+                                                                                           ignore_configuration_condition)).fetchone()[
+            0]
+    elif experiment_count == 'db_tries_best':
+        num_experiments = len(result_ids_for_parameters(experiment_name, best_params))
+    elif experiment_count == 'db_tries_initial':
+        num_experiments = len(result_ids_for_parameters(experiment_name, initial_params))
+    else:
+        raise ValueError('Invalid argument for experiment_count')
+    last_best_score = results_for_params(best_params)[1]
+    while num_experiments < max_num_experiments:
+
+        if num_experiments < initial_experiments:
+            try_params = initial_params.copy()
+        else:
+            any_results = \
+                cursor.execute('SELECT EXISTS (SELECT * FROM {0} WHERE NOT ({1}))'.format(experiment_name,
+                                                                                          ignore_configuration_condition)).fetchone()[
+                    0]
+            if any_results:
+                last_best_params = best_params
+                best_params = get_best_params(experiment_name,
+                                              larger_result_is_better,
+                                              optimize,
+                                              param_names,
+                                              additional_condition=f'NOT ({ignore_configuration_condition})')
+                best_scores, best_score, _, best_conf_size = results_for_params(best_params)
+                if last_best_score is not None and best_score is not None:
+                    if last_best_params != best_params:
+                        if last_best_score < best_score and larger_result_is_better or last_best_score > best_score and not larger_result_is_better:
+                            print(' --> Parameters were improved by this change!')
+                        if last_best_score > best_score and larger_result_is_better or last_best_score < best_score and not larger_result_is_better:
+                            print(' --> Actually other parameters are better...')
+                last_best_score = best_score
+
+                # print('currently best parameters:', best_params)
+                changed_params = {k: v for k, v in best_params.items() if best_params[k] != initial_params[k]}
+                print('currently best parameters (excluding unchanged parameters):', changed_params)
+                print('currently best score:', best_score, 'conf.', best_conf_size, 'num.', len(best_scores))
+            else:
+                best_params = {param.name: param.initial_value for param in params}
+                best_conf_size = inf
+            try_params = best_params.copy()
+            verbose = 1
+            if best_conf_size != inf:
+                if random.random() > no_mutations_probability:
+                    modify_params_randomly(mutation_probability, params, try_params, verbose,
+                                           allow_multiple_mutations=allow_multiple_mutations)
+
+        if num_experiments < initial_experiments:
+            try_params = initial_params.copy()
+        else:
+            # check if this already has a bad score
+            if skip():
+                print('skipping because this set of parameters is known to be worse with high probability.')
+                print()
+                continue
+
+        # print('trying parameters', {k: v for k, v in try_params.items() if try_params[k] != initial_params[k]})
+
+        results = try_(try_params)
+
+        if experiment_count == 'tries':
+            num_experiments += 1
+        elif experiment_count == 'results':
+            num_experiments += len(results)
+        elif experiment_count == 'db_total':
+            num_experiments = cursor.execute('SELECT COUNT(*) FROM {0}'.format(experiment_name)).fetchone()[0]
+        elif experiment_count == 'db_tries_best':
+            num_experiments = len(result_ids_for_parameters(experiment_name, best_params))
+        elif experiment_count == 'db_tries_initial':
+            num_experiments = len(result_ids_for_parameters(experiment_name, initial_params))
+        else:
+            raise LogicError('It is not possible that this is reached.')
+
+
+@with_goto
+def diamond_parameter_search(experiment_name: str,
+                             diamond_size: int,
+                             params: List[Parameter],
+                             evaluate: EvaluationFunction,
+                             optimize: str,
+                             larger_result_is_better: bool,
+                             runs_per_configuration=inf,
+                             initial_runs=1,
+                             metric_names=None,
+                             filter_results_condition='1'):
+    print('experiment name:', experiment_name)
+    if metric_names is None:
+        metric_names = [optimize]
+    if optimize not in metric_names:
+        raise ValueError('trying to optimize {0} but only metrics available are {1}'.format(optimize, metric_names))
+    print('Optimizing metric', optimize)
+    if runs_per_configuration > initial_runs:
+        print(
+            f'WARNING: You are using initial_runs={initial_runs} and runs_per_configuration={runs_per_configuration}. '
+            f'This may lead to unexpected results if you dont know what you are doing.')
+    params_in_original_order = params
+    params = sorted(params, key=lambda p: p.name)
+    validate_parameter_set(params)
+
+    create_experiment_tables_if_not_exists(experiment_name, params, metric_names)
+
+    initial_params = {param.name: param.initial_value for param in params}
+    print('initial parameters:', initial_params)
+
+    # get best params
+    initial_params = {param.name: param.initial_value for param in params}
+    try:
+        best_params = get_best_params_and_compare_with_initial(experiment_name, initial_params, larger_result_is_better,
+                                                               optimize,
+                                                               additional_condition=filter_results_condition)
+    except EmptyTableError:
+        best_params = initial_params
+
+    def max_tries_reached(ps):
+        return len(result_ids_for_parameters(experiment_name, ps)) >= runs_per_configuration
+
+    def min_tries_reached(ps):
+        return len(result_ids_for_parameters(experiment_name, ps)) >= initial_runs
+
+    def try_(ps) -> bool:
+        tried = False
+        if not max_tries_reached(ps):
+            try_parameters(experiment_name=experiment_name,
+                           evaluate=evaluate,
+                           params=ps,
+                           optimize=optimize,
+                           larger_result_is_better=larger_result_is_better, )
+            tried = True
+        else:
+            print('Skipping because maximum number of tries is already reached.')
+        while not min_tries_reached(ps):
+            print('Repeating because minimum number of tries is not reached.')
+            try_parameters(experiment_name=experiment_name,
+                           evaluate=evaluate,
+                           params=ps,
+                           optimize=optimize,
+                           larger_result_is_better=larger_result_is_better, )
+            tried = True
+        return tried
+
+    last_best_score = results_for_params(optimize, experiment_name, best_params)[1]
+    modifications_steps = [
+        {'param_name': param.name, 'direction': direction}
+        for param in params_in_original_order
+        for direction in ([param.larger_value, param.smaller_value] if param.first_try_increase
+                          else [param.smaller_value, param.larger_value])
+    ]
+    label.restart
+    restart_scheduled = False
+    while True:  # repeatedly iterate parameters
+        any_tries_done_this_iteration = False
+        for num_modifications in range(diamond_size + 1):  # first try small changes, later larger changes
+            modification_sets = itertools.product(*(modifications_steps for _ in range(num_modifications)))
+            for modifications in modification_sets:  # which modifications to try this time
+                while True:  # repeatedly modify parameters in this direction
+                    improvement_found_in_this_iteration = False
+                    try_params = best_params.copy()
+                    for modification in modifications:
+                        try_params[modification['param_name']] = modification['direction'](
+                            try_params[modification['param_name']])
+                    for param_name, param_value in try_params.items():
+                        if best_params[param_name] != param_value:
+                            print(f'Setting {param_name} = {param_value} for the next run.')
+                    if try_params == best_params:
+                        print('Repeating experiment with best found parameters.')
+
+                    if try_(try_params):  # if the experiment was actually conducted
+                        any_tries_done_this_iteration = True
+                        best_params = get_best_params_and_compare_with_initial(experiment_name, initial_params,
+                                                                               larger_result_is_better, optimize,
+                                                                               filter_results_condition)
+
+                        last_best_params = best_params
+                        best_scores, best_score, _, best_conf_size = results_for_params(optimize, experiment_name,
+                                                                                        best_params)
+
+                        changed_params = {k: v for k, v in best_params.items() if best_params[k] != initial_params[k]}
+                        print('currently best parameters (excluding unchanged parameters):', changed_params)
+                        print('currently best score:', best_score, 'conf.', best_conf_size, 'num.', len(best_scores))
+                    else:
+                        last_best_params = best_params
+                        _, best_score, _, best_conf_size = results_for_params(optimize, experiment_name, best_params)
+
+                    if last_best_score is not None and best_score is not None:
+                        if last_best_params != best_params:
+                            if last_best_score < best_score and larger_result_is_better or last_best_score > best_score and not larger_result_is_better:
+                                print(' --> Parameters were improved by this change!')
+                                improvement_found_in_this_iteration = True
+                                if num_modifications > 1:
+                                    # two or more parameters were modified and this improved the results -> first try to modify them again in the same direction,
+                                    # then restart the search from the best found configuration
+                                    restart_scheduled = True
+                            elif last_best_score > best_score and larger_result_is_better or last_best_score < best_score and not larger_result_is_better:
+                                print(' --> Actually other parameters are better...')
+                    if not improvement_found_in_this_iteration:
+                        break  # stop if no improvement was found in this direction
+                if restart_scheduled:
+                    break
+            if restart_scheduled:
+                break
+        if restart_scheduled:
+            goto.restart
+        if not any_tries_done_this_iteration:
+            break  # parameter search finished (converged in some sense)
+
+
+cross_parameter_search = functools.partial(diamond_parameter_search, diamond_size=1)
+cross_parameter_search.__name__ = 'cross_parameter_search'
+
+
+def get_best_params_and_compare_with_initial(experiment_name, initial_params, larger_result_is_better, optimize,
+                                             additional_condition='1'):
+    best_params = get_best_params(experiment_name, larger_result_is_better, optimize, list(initial_params),
+                                  additional_condition=additional_condition)
+    changed_params = {k: v for k, v in best_params.items() if best_params[k] != initial_params[k]}
+    best_scores, best_score, _, best_conf_size = results_for_params(optimize, experiment_name, best_params)
+    print('currently best parameters (excluding unchanged parameters):', changed_params)
+    print('currently best score:', best_score, 'conf.', best_conf_size, 'num.', len(best_scores))
+    return best_params
+
+
+def results_for_params(optimize, experiment_name, ps):
+    return get_results_for_params(
+        metric=optimize,
+        experiment_name=experiment_name,
+        parameters=ps
+    )
+
+
+def modify_params_randomly(mutation_probability, params, try_params, verbose, allow_multiple_mutations=False):
+    for param in params:
+        while random.random() < mutation_probability:
+            next_value = random.choice([param.smaller_value, param.larger_value])
+            old_value = try_params[param.name]
+            try:
+                try_params[param.name] = round_to_digits(next_value(try_params[param.name]), 4)
+            except TypeError:  # when the parameter is not a number
+                try_params[param.name] = next_value(try_params[param.name])
+            if verbose and try_params[param.name] != old_value:
+                print('setting', param.name, '=', try_params[param.name], 'for this run')
+            if not allow_multiple_mutations:
+                break
+
+
+def finish_experiments(experiment_name: str,
+                       params: List[Parameter],
+                       optimize: str,
+                       larger_result_is_better: bool,
+                       metric_names=None,
+                       filter_results_table='1',
+                       max_display_results=None,
+                       print_results_table=False,
+                       max_table_row_count=inf,
+                       plot_metrics_by_metrics=False,
+                       plot_metric_over_time=False,
+                       plot_metrics_by_parameters=False, ):
+    if max_display_results is inf:
+        max_display_results = None
+    if metric_names is None:
+        metric_names = [optimize]
+    # get the best parameters
+    cursor = connection.cursor()
+    params = sorted(params, key=lambda param: param.name)
+    param_names = sorted(set(param.name for param in params))
+    param_names_comma_separated = ','.join('"' + param_name + '"' for param_name in param_names)
+
+    best_params = get_best_params(experiment_name, larger_result_is_better, optimize, param_names,
+                                  additional_condition=filter_results_table, )
+    best_score = get_results_for_params(
+        metric=optimize,
+        experiment_name=experiment_name,
+        parameters=best_params
+    )
+    initial_params = {param.name: param.initial_value for param in params}
+
+    # get a list of all results with mean std and conf
+    if print_results_table or plot_metrics_by_parameters or plot_metrics_by_metrics:
+        concatenated_metric_names = ','.join('GROUP_CONCAT("' + metric_name + '", \'@\') AS ' + metric_name
+                                             for metric_name in metric_names)
+        worst_score = '-1e999999' if larger_result_is_better else '1e999999'
+        limit_string = f'LIMIT {max_table_row_count}' if max_table_row_count is not None and max_table_row_count < inf else ''
+        # noinspection SqlAggregates
+        cursor.execute('''
+        SELECT {1}, {4}
+        FROM {0} AS params
+        WHERE ({5})
+        GROUP BY {1}
+        ORDER BY AVG(CASE WHEN params.{3} IS NULL THEN {6} ELSE params.{3} END) {2}
+        {7}
+        '''.format(experiment_name,
+                   param_names_comma_separated,
+                   'DESC' if larger_result_is_better else 'ASC',
+                   optimize,
+                   concatenated_metric_names,
+                   filter_results_table,
+                   worst_score,
+                   limit_string))
+        all_results = cursor.fetchall()
+
+        column_description = list(cursor.description)
+
+        for idx, row in enumerate(all_results):
+            all_results[idx] = list(row)
+
+        # prepare results table
+        if print_results_table or plot_metrics_by_metrics or plot_metrics_by_parameters:
+            iterations = 0
+            print('Generating table of parameters')
+            for column_index, column in list(enumerate(column_description))[::-1]:  # reverse
+                print_progress_bar(iterations, len(metric_names))
+                column_name = column[0]
+                column_description[column_index] = column
+                if column_name in metric_names:
+                    if max_display_results > 0:
+                        column_description[column_index] = column_name + ' values'
+                    column_description.insert(column_index + 1, column_name + ' mean')
+                    column_description.insert(column_index + 2, column_name + ' std')
+                    column_description.insert(column_index + 3, column_name + ' conf')
+                    # noinspection PyUnusedLocal
+                    list_row: List
+                    for list_row in all_results:
+                        string_values: str = list_row[column_index]
+                        if string_values is None:
+                            metric_values: List[float] = [nan]
+                        else:
+                            metric_values = list(map(float, string_values.split('@')))
+                        list_row[column_index] = [round_to_digits(x, 3) for x in metric_values[:max_display_results]]
+                        list_row.insert(column_index + 1, numpy.mean(metric_values))
+                        list_row.insert(column_index + 2, numpy.std(metric_values))
+                        list_row.insert(column_index + 3, mean_confidence_interval_size(metric_values))
+                    if all(len(list_row[column_index]) == 0 for list_row in all_results):
+                        del column_description[column_index]
+                        for list_row in all_results:
+                            del list_row[column_index]
+                    iterations += 1
+                else:
+                    column_description[column_index] = column_name
+                print_progress_bar(iterations, len(metric_names))
+
+        if print_results_table:  # actually print the table
+            table = my_tabulate(all_results,
+                                headers=column_description,
+                                tablefmt='pipe')
+
+            print(table)
+            cursor.execute('''
+            SELECT COUNT(*)
+            FROM {0}
+            '''.format(experiment_name))
+            print('Total number of rows, experiments, cells in this table:',
+                  (len(all_results), cursor.fetchone()[0], len(all_results) * len(all_results[0])))
+            print('Best parameters:', best_params)
+            changed_params = {k: v for k, v in best_params.items() if best_params[k] != initial_params[k]}
+            print('Best parameters (excluding unchanged parameters):', changed_params)
+            print('loss/score for best parameters (mean, std, conf):', best_score[1:])
+
+        if plot_metrics_by_parameters or plot_metrics_by_metrics:
+            print('Loading data...')
+            df = pandas.DataFrame.from_records(all_results, columns=param_names + [x
+                                                                                   for name in metric_names
+                                                                                   for x in
+                                                                                   [
+                                                                                       name + '_values',
+                                                                                       name + '_mean',
+                                                                                       name + '_std',
+                                                                                       name + '_conf'
+                                                                                   ]])
+            if plot_metrics_by_parameters:
+                print('Plotting metrics by parameter...')
+                plots = [
+                    (param.name,
+                     getattr(param, 'plot_scale', None),
+                     param.smaller_value if isinstance(param, BoundedParameter) else None,
+                     param.larger_value if isinstance(param, BoundedParameter) else None)
+                    for param in params
+                ]
+
+                iterations = 0
+                for metric_name in metric_names:
+                    dirname = 'img/results/{0}/{1}/'.format(experiment_name, metric_name)
+                    os.makedirs(dirname, exist_ok=True)
+                    for plot, x_scale, min_mod, max_mod in plots:
+                        print_progress_bar(iterations, len(metric_names) * len(plots))
+                        if min_mod is None:
+                            min_mod = lambda x: x
+                        if max_mod is None:
+                            max_mod = lambda x: x
+                        if df[plot].nunique() <= 1:
+                            iterations += 1
+                            continue
+                        grid = sns.relplot(x=plot, y=metric_name + '_mean', data=df)
+                        if x_scale is not None:
+                            if x_scale == 'log' and min_mod(df.min(axis=0)[plot]) <= 0:
+                                x_min = None
+                            else:
+                                x_min = min_mod(df.min(axis=0)[plot])
+                            grid.set(xscale=x_scale,
+                                     xlim=(x_min,
+                                           max_mod(df.max(axis=0)[plot]),))
+                        plt.savefig(dirname + '{0}.png'.format(plot))
+                        plt.clf()
+                        plt.close()
+                        iterations += 1
+                        print_progress_bar(iterations, len(metric_names) * len(plots))
+
+            if plot_metrics_by_metrics:
+                print('Plotting metrics by metrics...')
+                dirname = 'img/results/{0}/'.format(experiment_name)
+                os.makedirs(dirname, exist_ok=True)
+
+                # Generate some plots, metric by metric
+                iterations = 0
+                print('Plotting metric by metric, grouped')
+                for metric_name in metric_names:
+                    for metric_2 in metric_names:
+                        if metric_name == metric_2:
+                            iterations += 1
+                            print_progress_bar(iterations, len(metric_names) ** 2)
+                            continue
+                        print_progress_bar(iterations, len(metric_names) ** 2)
+                        sns.relplot(x=metric_name + '_mean', y=metric_2 + '_mean', data=df)
+                        plt.savefig(dirname + '{0}_{1}_grouped.png'.format(metric_name, metric_2))
+                        plt.clf()
+                        plt.close()
+                        heatmap_from_points(x=df[metric_name + '_mean'], y=df[metric_2 + '_mean'])
+                        plt.xlabel(f'mean {metric_name}')
+                        plt.ylabel(f'mean {metric_2}')
+                        plt.savefig(dirname + '{0}_{1}_heatmap.png'.format(metric_name, metric_2))
+                        plt.clf()
+                        plt.close()
+                        iterations += 1
+                        print_progress_bar(iterations, len(metric_names) ** 2)
+
+                df = pandas.read_sql_query('SELECT * FROM {0}'.format(experiment_name),
+                                           connection)
+                df['dt_created'] = pandas.to_datetime(df['dt_created'])
+
+            if plot_metric_over_time:
+                # Generate some plots, metric over time
+                dirname = 'img/results/{0}/'.format(experiment_name)
+                os.makedirs(dirname, exist_ok=True)
+                print('Plotting metric over time')
+                iterations = 0
+                for metric_name in metric_names:
+                    if not df[metric_name].any():
+                        continue
+                    print_progress_bar(iterations, len(metric_names))
+                    ax = df.plot(x='dt_created', y=metric_name, style='.')
+                    ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter('%m-%d %H:00'))
+                    plt.savefig(dirname + 'dt_created_{0}.png'.format(metric_name))
+                    plt.clf()
+                    plt.close()
+                    iterations += 1
+                    print_progress_bar(iterations, len(metric_names))
+
+                # plot optimize grouped over time
+                assert df['dt_created'].is_monotonic  # sorting should not be a problem but we are lazy
+                y_means = []
+                df = df.drop_duplicates(subset='dt_created')
+                timestamps = pandas.datetimeIndex(df.dt_created).asi8 // 10 ** 9
+                iterations = 0
+                print('Preparing plot {0} over time'.format(optimize))
+                for x in timestamps:
+                    print_progress_bar(iterations, len(timestamps))
+                    not_after_x = 'CAST(strftime(\'%s\', dt_created) AS INT) <= {0}'.format(x)
+                    param = get_best_params(additional_condition=not_after_x,
+                                            param_names=param_names,
+                                            experiment_name=experiment_name,
+                                            larger_result_is_better=larger_result_is_better,
+                                            optimize=optimize)
+                    scores, mean, std, conf = get_results_for_params(optimize, experiment_name, param,
+                                                                     additional_condition=not_after_x)
+                    y_means.append(mean)
+                    iterations += 1
+                    print_progress_bar(iterations, len(timestamps))
+                df['score'] = y_means
+
+                ax = df.plot(x='dt_created', y='score')
+                ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter('%m-%d %H:00'))
+                plt.savefig(dirname + '{0}_over_time.png'.format(optimize))
+                plt.clf()
+                plt.close()
+
+    return best_params, best_score
+
+
+def predictions_for_parameters(experiment_name: str, parameters, show_progress=False):
+    result_ids = result_ids_for_parameters(experiment_name, parameters)
+    if not show_progress:
+        return [
+            predictions_for_result_id(experiment_name, result_id)
+            for result_id in result_ids
+        ]
+    else:
+        return [
+            predictions_for_result_id(experiment_name, result_id)
+            for result_id in ProgressBar(result_ids)
+        ]
+
+
+def result_ids_for_parameters(experiment_name, parameters: Dict[str, Any]):
+    condition, parameters = only_specific_parameters_condition(parameters)
+    cursor = connection.cursor()
+    cursor.execute('''
+    SELECT rowid FROM {0}
+    WHERE {1}
+    ORDER BY rowid
+    '''.format(experiment_name, condition), parameters)
+    result_ids = [row[0] for row in cursor.fetchall()]
+    return result_ids
+
+
+def creation_times_for_parameters(experiment_name, parameters):
+    condition, parameters = only_specific_parameters_condition(parameters)
+    cursor = connection.cursor()
+    cursor.execute('''
+    SELECT dt_created FROM {0}
+    WHERE {1}
+    ORDER BY rowid
+    '''.format(experiment_name, condition), parameters)
+    creation_times = [row[0] for row in cursor.fetchall()]
+    return creation_times
+
+
+def predictions_for_result_id(experiment_name: str, result_id):
+    cursor = connection.cursor()
+    cursor.execute('''
+    SELECT name, dataset, y_pred, y_true FROM {0}_predictions
+    WHERE result_id = ?
+    '''.format(experiment_name, ), (result_id,))
+    predictions = [{
+        'name': row[0],
+        'dataset': row[1],
+        'y_pred': row[2],
+        'y_true': row[3],
+    } for row in cursor.fetchall()]
+
+    return predictions
+
+
+def list_difficult_samples(experiment_name,
+                           loss_functions,
+                           dataset,
+                           max_losses_to_average=20,
+                           additional_condition='1',
+                           additional_parameters=(),
+                           also_print=False):
+    names = all_sample_names(dataset, experiment_name)
+    cursor = connection.cursor()
+    if 'epochs' in additional_condition:
+        try:
+            print('Creating index to fetch results faster (if not exists)...')
+            cursor.execute('''
+                CREATE INDEX IF NOT EXISTS {0}_by_name_epochs_dataset
+                ON {0} (name, epochs, dataset)'''.format(experiment_name))
+        except Exception as e:  # TODO check error type
+            print(e)
+            pass
+    cursor = connection.cursor()
+    table = []
+    print('Fetching results for names...')
+    for name in ProgressBar(names):
+        if additional_condition == '1':
+            additional_join = ''
+        else:
+            additional_join = 'JOIN {0} ON {0}.rowid = result_id'.format(experiment_name)
+        if isinstance(max_losses_to_average, int) is not None and max_losses_to_average != inf:
+            limit_string = f'LIMIT ?'
+            limit_args = [max_losses_to_average]
+        elif max_losses_to_average is None or max_losses_to_average == inf:
+            limit_string = ''
+            limit_args = []
+        else:
+            raise ValueError
+        cursor.execute('''
+            SELECT y_pred, y_true
+            FROM {0}
+             CROSS JOIN {0}_predictions ON {0}.rowid = result_id
+            WHERE name = ? AND dataset = ? AND ({1})
+            {3}'''.format(experiment_name, additional_condition, ..., limit_string), (name,
+                                                                                      dataset,
+                                                                                      *additional_parameters,
+                                                                                      *limit_args,))
+        data = cursor.fetchall()
+        if len(data) > 0:
+            def aggregate(xs):
+                if len(set(xs)) == 1:
+                    return xs[0]
+                else:
+                    return numpy.mean(xs)
+
+            table.append((*[aggregate([loss_function(y_pred=y_pred, y_true=y_true, name=name)
+                                       for y_pred, y_true in data])
+                            for loss_function in loss_functions],
+                          name, len(data)))
+    print('sorting table...')
+    table.sort(reverse=True)
+    if also_print:
+        print('stringifying table...')
+        print(my_tabulate(table,
+                          headers=[loss_function.__name__ for loss_function in loss_functions] + ['name', '#results'],
+                          tablefmt='pipe'))
+    return table
+
+
+def all_sample_names(dataset, experiment_name):
+    cursor = connection.cursor()
+    print('Creating index to have faster queries by name (if not exists)...')
+    cursor.execute('''
+        CREATE INDEX IF NOT EXISTS {0}_predictions_by_name_and_dataset
+        ON {0}_predictions (dataset, name)'''.format(experiment_name))
+    print('Fetching all names...')
+    names = []
+    last_found = ''  # smaller than all other strings
+    while True:
+        cursor.execute('SELECT name '
+                       'FROM {0}_predictions '
+                       'WHERE dataset = ? AND name > ?'
+                       'LIMIT 1'.format(experiment_name), (dataset, last_found))
+        row = cursor.fetchone()
+        if row is None:
+            break
+        names.append(row[0])
+        last_found = row[0]
+    return names
+
+
+def only_specific_parameters_condition(parameters: Dict[str, Any]) -> Tuple[str, Tuple]:
+    items = list(parameters.items())  # to have the correct ordering
+    return '(' + ' AND '.join(f'"{name}" IS ?' for name, _ in items) + ')', \
+           tuple(value for name, value in items)
+
+
+def only_best_parameters_condition(experiment_name: str,
+                                   larger_result_is_better: bool,
+                                   optimize: str,
+                                   param_names: List[str],
+                                   additional_condition: str = '1') -> Tuple[str, Tuple]:
+    parameters = get_best_params(experiment_name=experiment_name,
+                                 larger_result_is_better=larger_result_is_better,
+                                 optimize=optimize,
+                                 param_names=param_names,
+                                 additional_condition=additional_condition)
+    return only_specific_parameters_condition(parameters)
+
+
+def get_results_for_params(metric, experiment_name, parameters, confidence=0.95,
+                           additional_condition='1'):
+    param_names = list(parameters.keys())
+    cursor = connection.cursor()
+    params_equal = '\nAND '.join('"' + param_name + '" IS ?' for param_name in param_names)
+    cursor.execute(
+        '''
+        SELECT {0}
+        FROM {1}
+        WHERE {2} AND ({3})
+        '''.format(metric,
+                   experiment_name,
+                   params_equal,
+                   additional_condition),
+        tuple(parameters[name] for name in param_names)
+    )
+    # noinspection PyShadowingNames
+    scores = [row[0] if row[0] is not None else nan for row in cursor.fetchall()]
+    if len(scores) == 0:
+        return scores, nan, nan, nan
+    return scores, numpy.mean(scores), numpy.std(scores), mean_confidence_interval_size(scores, confidence)
+
+
+def num_results_for_params(param_names, experiment_name, parameters,
+                           additional_condition='1'):
+    cursor = connection.cursor()
+    params_equal = '\nAND '.join('"' + param_name + '" IS ?' for param_name in param_names)
+    cursor.execute(
+        '''
+        SELECT COUNT(*)
+        FROM {0}
+        WHERE {1} AND ({2})
+        '''.format(experiment_name,
+                   params_equal,
+                   additional_condition),
+        tuple(parameters[name] for name in param_names)
+    )
+    return cursor.fetchone()[0]
+
+
+def get_best_params(experiment_name: str,
+                    larger_result_is_better: bool,
+                    optimize: str,
+                    param_names: List[str],
+                    additional_condition='1') -> Optional[Parameters]:
+    cursor = connection.cursor()
+    param_names_comma_separated = ','.join('"' + param_name + '"' for param_name in param_names)
+    worst_score = '-1e999999' if larger_result_is_better else '1e999999'
+    # noinspection SqlAggregates
+    cursor.execute('''
+            SELECT * FROM {0} AS params
+            WHERE ({4})
+            GROUP BY {1}
+            ORDER BY AVG(CASE WHEN params.{3} IS NULL THEN {5} ELSE params.{3} END) {2}, MIN(rowid) ASC
+            LIMIT 1
+            '''.format(experiment_name,
+                       param_names_comma_separated,
+                       'DESC' if larger_result_is_better else 'ASC',
+                       optimize,
+                       additional_condition,
+                       worst_score, ))
+    row = cursor.fetchone()
+    if row is None:
+        raise EmptyTableError()
+    else:
+        return params_from_row(cursor.description, row, param_names=param_names)
+
+
+def params_from_row(description, row, param_names=None) -> Parameters:
+    best_params = {}
+    for idx, column_description in enumerate(description):
+        column_name = column_description[0]
+        if param_names is None or column_name in param_names:
+            best_params[column_name] = row[idx]
+    return best_params
+
+
+def create_experiment_tables_if_not_exists(experiment_name, params, metric_names):
+    cursor = connection.cursor()
+    param_names = set(param.name for param in params)
+    initial_params = {param.name: param.initial_value for param in params}
+    cursor.execute('''
+    CREATE TABLE IF NOT EXISTS {0}(
+        rowid INTEGER PRIMARY KEY,
+        dt_created datetime DEFAULT CURRENT_TIMESTAMP
+    )
+    '''.format(experiment_name))
+    cursor.execute('''
+    CREATE TABLE IF NOT EXISTS {0}_predictions(
+        rowid INTEGER PRIMARY KEY,
+        dataset TEXT NOT NULL,
+        y_true BLOB,
+        y_pred BLOB,
+        name TEXT NOT NULL, -- used to identify the samples
+        result_id INTEGER NOT NULL REFERENCES {0}(rowid),
+        UNIQUE(result_id, dataset, name) -- gives additional indices
+    )
+    '''.format(experiment_name))
+    connection.commit()
+
+    for param_name in param_names:
+        default_value = initial_params[param_name]
+        add_parameter_column(experiment_name, param_name, default_value)
+
+    for metric_name in metric_names:
+        add_metric_column(experiment_name, metric_name)
+
+
+def add_metric_column(experiment_name, metric_name, verbose=0):
+    cursor = connection.cursor()
+    try:
+        cursor.execute('ALTER TABLE {0} ADD COLUMN "{1}" NUMERIC DEFAULT NULL'.format(experiment_name,
+                                                                                      metric_name))
+    except sqlite3.OperationalError as e:
+        if 'duplicate column name' not in e.args[0]:
+            raise
+    else:
+        if verbose:
+            print(f'WARNING: created additional column {metric_name}. This may or may not be intentional')
+    connection.commit()
+
+
+def add_parameter_column(experiment_name, param_name, default_value, verbose=0):
+    cursor = connection.cursor()
+    try:
+        if isinstance(default_value, str):
+            default_value.replace("'", "\\'")
+            default_value = "'" + default_value + "'"
+        if default_value is None:
+            default_value = 'NULL'
+        cursor.execute('ALTER TABLE {0} ADD COLUMN "{1}" BLOB DEFAULT {2}'.format(experiment_name,
+                                                                                  param_name,
+                                                                                  default_value))
+    except sqlite3.OperationalError as e:
+        if 'duplicate column name' not in e.args[0]:
+            raise
+    else:
+        if verbose:
+            print(
+                f'WARNING: created additional column {param_name} with default value {default_value}. This may or may not be intentional')
+    connection.commit()
+
+
+def markdown_table(all_results, sort_by):
+    rows = [list(result['params'].values()) + [result['mean'], result['std'], result['conf'], result['all']] for result
+            in all_results]
+    rows.sort(key=sort_by)
+    table = my_tabulate(rows, headers=list(all_results[0]['params'].keys()) + ['mean', 'std', 'conf', 'results'],
+                        tablefmt='pipe')
+    return table
+
+
+def validate_parameter_set(params):
+    if len(params) == 0:
+        raise ValueError('Parameter set empty')
+    for i, param in enumerate(params):
+        # noinspection PyUnusedLocal
+        other_param: Parameter
+        for other_param in params[i + 1:]:
+            if param.name == other_param.name and param.initial_value != other_param.initial_value:
+                msg = '''
+                A single parameter cant have multiple initial values. 
+                Parameter "{0}" has initial values "{1}" and "{2}"
+                '''.format(param.name, param.initial_value, other_param.initial_value)
+                raise ValueError(msg)
+
+
+def run_name(parameters=None) -> str:
+    if parameters is None:
+        parameters = {}
+    shorter_parameters = {
+        shorten_name(k): shorten_name(v)
+        for k, v in parameters.items()
+    }
+    return ((str(datetime.now()) + str(shorter_parameters).replace(' ', ''))
+            .replace("'", '')
+            .replace('"', '')
+            .replace(":", '⦂')
+            .replace(",", '')
+            .replace("_", '')
+            .replace("<", '')
+            .replace(">", '')
+            .replace("{", '')
+            .replace("}", ''))
+
+
+def plot_experiment(metric_names,
+                    experiment_name: str,
+                    plot_name: str,
+                    param_names: List[str],
+                    params_list: List[Parameters],
+                    evaluate: EvaluationFunction,
+                    ignore: List[str] = None,
+                    plot_shape=None,
+                    metric_limits: Dict = None,
+                    titles=None,
+                    natural_metric_names: Dict[str, str] = None,
+                    min_runs_per_params=0,
+                    single_plot_width=6.4,
+                    single_plot_height=4.8, ):
+    if natural_metric_names is None:
+        natural_metric_names = {}
+    for parameters in params_list:
+        if 'epochs' not in parameters:
+            raise ValueError('`plot_experiment` needs the number of epochs to plot (`epochs`)')
+
+    if metric_limits is None:
+        metric_limits = {}
+    if ignore is None:
+        ignore = []
+    if titles is None:
+        titles = [None for _ in params_list]
+
+    if plot_shape is None:
+        width = ceil(sqrt(len(params_list)))
+        plot_shape = (ceil(len(params_list) / width), width,)
+    else:
+        width = plot_shape[1]
+    plot_shape_offset = 100 * plot_shape[0] + 10 * plot_shape[1]
+    axes: Dict[int, Axes] = {}
+    legend: List[str] = []
+    results_dir = 'img/results/{0}/over_time/'.format(experiment_name)
+    os.makedirs(results_dir, exist_ok=True)
+
+    metric_names = sorted(metric_names, key=lambda m: (metric_limits.get(m, ()), metric_names.index(m)))
+    print(metric_names)
+    plotted_metric_names = []
+    iterations = 0
+
+    for plot_std in [False, True]:
+        plt.figure(figsize=(single_plot_width * plot_shape[1], single_plot_height * plot_shape[0]))
+        for idx, metric in enumerate(metric_names):
+            print_progress_bar(iterations, 2 * (len(metric_names) - len(ignore)))
+            limits = metric_limits.get(metric, None)
+            try:
+                next_limits = metric_limits.get(metric_names[idx + 1], None)
+            except IndexError:
+                next_limits = None
+            if metric in ignore:
+                continue
+            sqlite_infinity = '1e999999'
+            metric_is_finite = '{0} IS NOT NULL AND {0} != {1} AND {0} != -{1}'.format(metric, sqlite_infinity)
+            for plot_idx, parameters in enumerate(params_list):
+                while num_results_for_params(param_names=param_names,
+                                             experiment_name=experiment_name,
+                                             parameters=parameters, ) < min_runs_per_params:
+                    print('Doing one of the missing experiments for the plot:')
+                    print(parameters)
+                    results = try_parameters(experiment_name=experiment_name,
+                                             evaluate=evaluate,
+                                             params=parameters, )
+                    assert any(result.parameters == parameters for result in results)
+
+                contains_avg_over = 'average_over_last_epochs' in parameters
+
+                total_epochs = parameters['epochs']
+                history = []
+                lower_conf_limits = []
+                upper_conf_limits = []
+                for epoch_end in range(total_epochs):
+                    current_parameters = parameters.copy()
+                    if contains_avg_over:
+                        current_parameters['average_over_last_epochs'] = None
+                    current_parameters['epochs'] = epoch_end + 1
+                    scores, mean, std, conf = get_results_for_params(
+                        metric=metric,
+                        experiment_name=experiment_name,
+                        parameters=current_parameters,
+                        additional_condition=metric_is_finite
+                    )
+                    history.append(mean)
+                    if plot_std:
+                        lower_conf_limits.append(mean - 1.959964 * std)
+                        upper_conf_limits.append(mean + 1.959964 * std)
+                    else:
+                        lower_conf_limits.append(mean - conf)
+                        upper_conf_limits.append(mean + conf)
+                x = list(range(len(history)))
+                if plot_shape_offset + plot_idx + 1 not in axes:
+                    # noinspection PyTypeChecker
+                    ax: Axes = plt.subplot(plot_shape_offset + plot_idx + 1)
+                    assert isinstance(ax, Axes)
+                    axes[plot_shape_offset + plot_idx + 1] = ax
+                ax = axes[plot_shape_offset + plot_idx + 1]
+                ax.plot(x, history)
+                ax.fill_between(x, lower_conf_limits, upper_conf_limits, alpha=0.4)
+                if titles[plot_idx] is not None:
+                    ax.set_title(titles[plot_idx])
+                if limits is not None:
+                    ax.set_ylim(limits)
+                ax.set_xlim(0, max(total_epochs, ax.get_xlim()[1]))
+                current_row = plot_idx // width
+                if current_row == plot_shape[0] - 1:
+                    ax.set_xlabel('Epoch')
+            natural_name = natural_metric_names.get(metric, metric)
+            if plot_std:
+                legend += ['mean ' + natural_name, '1.96σ of {0}'.format(natural_name)]
+            else:
+                legend += ['mean ' + natural_name, '95% conf. of mean {0}'.format(natural_name)]
+
+            plotted_metric_names.append(metric)
+            if limits is None or next_limits is None or limits != next_limits:
+                legend = legend[0::2] + legend[1::2]
+                for ax in axes.values():
+                    ax.legend(legend)
+                if plot_std:
+                    plt.savefig(results_dir + plot_name + '_' + ','.join(plotted_metric_names) + '_std' + '.png')
+                else:
+                    plt.savefig(results_dir + plot_name + '_' + ','.join(plotted_metric_names) + '.png')
+                plt.clf()
+                plt.close()
+                plt.figure(figsize=(single_plot_width * plot_shape[1], single_plot_height * plot_shape[0]))
+                axes = {}
+                plotted_metric_names = []
+                legend = []
+            iterations += 1
+            print_progress_bar(iterations, 2 * (len(metric_names) - len(ignore)))
+    plt.clf()
+    plt.close()
+
+
+if __name__ == '__main__':
+    def evaluate(params):
+        return (params['A'] - 30) ** 2 + 10 * ((params['B'] / (params['A'] + 1)) - 1) ** 2 + params['C']
+
+
+    diamond_parameter_search('test',
+                             diamond_size=2,
+                             params=[LinearParameter('A', 10, 10),
+                                     ExponentialIntegerParameter('B', 8, 2),
+                                     ConstantParameter('C', 5)],
+                             runs_per_configuration=1,
+                             initial_runs=1,
+                             evaluate=evaluate,
+                             optimize='loss',
+                             larger_result_is_better=False)

tool_lib/path.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+from os import getcwd
+class Path_Lib:
+    def __init__(self,name='',path=''):
+        self.name = name
+        self.path = path
+        pass
+    # Low Level
+    def get_cwd_path(self):
+        return getcwd()
+    # Mid Level
+    def get_path_of_file_in_cwd(self, name=''):
+        if name =='':
+            name = self.name
+        path= getcwd()+ '\\'+ name
+        return path
+

tool_lib/plots.py

@@ -0,0 +1,221 @@
+import sqlite3
+import time
+from typing import Union
+import networkx as nx
+import matplotlib.pyplot as plt
+import numpy as np
+import sys
+import matplotlib.animation
+
+
+class Plots():
+    def __init__(self, x_axe, y_axe, z_axe=[], x_label='x', y_label='y', z_label='z', x_axe2=[], y_axe2=[], z_axe2=[], grid=True,
+                 figsize=(13, 10)):
+        self.x_axe = x_axe
+        self.y_axe = y_axe
+        self.z_axe = z_axe
+        self.x_label = x_label
+        self.y_label = y_label
+        self.z_label = z_label
+        self.x_axe2 = x_axe2
+        self.y_axe2 = y_axe2
+        self.z_axe2 = z_axe2
+        self.grid = grid
+        self.figure = plt.figure(figsize=figsize)
+
+    def get_a_exponential_func_as_plot(self, linrange=[-5, 5], expo=2):
+        x = np.linspace(linrange[0], linrange[1])
+        y = x ** expo
+        plt.plot(x, y, 'g')
+
+    def plot_2D_compare_bar_chart(self, legend=[], width=0.35, title=str, path=''):
+        x = np.arange(len(self.x_axe))
+
+        if legend:
+            plt.bar(x, self.y_axe, width=width, color='blue', label=legend[0])
+            plt.bar(x + width, self.y_axe2, width=width, color='red', label=legend[1])
+        else:
+            plt.bar(x, self.y_axe, width=width, color='blue')
+            plt.bar(x + width, self.y_axe2, width=width, color='red')
+
+        if self.grid:
+            plt.grid()
+
+        plt.xlabel(self.x_label)
+        plt.ylabel(self.y_label)
+
+        if title:
+            plt.title(title)
+
+        plt.xticks(x + width / 2, self.x_axe)
+        plt.legend(loc='best')
+
+        if path:
+            self.save_fig(name='plot_2D_compare_bar_chart', path=path)
+        else:
+            self.save_fig(name='plot_2D_compare_bar_chart')
+        plt.show()
+
+    def save_fig(self, name, path: Union[bool, str] = False, ):
+        if path:
+            plt.savefig(path + '{}_{}.png'.format(name, time.strftime("%Y-%m-%d_H%H-M%M")))
+        else:
+            plt.savefig('{}_{}.png'.format(name, time.strftime("%Y-%m-%d_H%H-M%M")))
+
+
+# Constant Layout
+NODE_SIZE = 200
+NODE_EDGE_COLOR = 'black'
+EDGE_WIDTH = 0.5
+FONT_SIZE = 8
+FONT_SIZE_EDGES = 1
+FONT_FAMILY = 'sans-serif'
+SAVE_FORMAT = 'svg'
+DPI = 1200
+CONNECTION_STYLE = 'arc3, rad=0.2'
+ARROW_SIZE = 12
+LABLE_POS = 0.35
+
+
+class NetworkxPlots():
+    def __init__(self, node_dict, pathing=[], color_map=[], legend=[], edges=[], edge_weightings=[], directed_graph=False):
+        if directed_graph:
+            self.graph = nx.DiGraph()
+        else:
+            self.graph = nx.Graph()
+
+        # Nodes
+        self.node_type_dict = {}
+        self.node_color_dict = {}
+        self.node_pos_dict = {}
+
+        for key, value in node_dict.items():
+            for ckey, cvalue in node_dict.items():
+                if node_dict[key]['node_type'] == node_dict[ckey]['node_type'] and node_dict[key][
+                    'node_type'] not in self.node_type_dict.keys():
+                    self.node_type_dict[node_dict[key]['node_type']] = []
+            self.node_type_dict[node_dict[key]['node_type']].append(key)
+            self.node_color_dict[node_dict[key]['node_type']] = node_dict[key]['node_color']
+            self.node_pos_dict[key] = node_dict[key]['node_name']
+
+        # Edges can be a nxn Matrix
+        self.edges = edges
+        self.pathing = pathing
+        self.color_map = color_map
+        self.edge_weightings = edge_weightings
+        self.legend = legend
+
+    def edges_for_complete_graph(self):
+        # without self_loops
+        for row in range(0, len(self.edges[:, 0])):
+            for column in range(0, len(self.edges[0, :])):
+                if round(self.edges[row, column], 1) == 0:
+                    pass
+                else:
+                    self.graph.add_edge(row + 1, column + 1, weight=1)  # round(self.edges[row, column], 1)
+
+    def directions_for_directed_graph(self, pathing=[]):
+        if pathing:
+            for order, path in enumerate(pathing):
+                self.graph.add_edge(path[0], path[1], weight=order + 1)
+        else:
+            for order, path in enumerate(self.pathing):
+                self.graph.add_edge(path[0], path[1], weight=order + 1)
+
+    def add_nodes_to_graph(self):
+        node_numberation = [node_number for node_number in range(1, len(self.node_pos_dict.keys()) + 1)]
+        for count in node_numberation:
+            self.graph.add_node(count)
+
+    def add_edges_to_graph(self):
+        self.graph.add_edges_from(self.edges, weight=self.edge_weightings)
+
+    def undirected_graph_plt_(self,
+                              name_path_tupl=''):  # TODO SIMPLYFY THESE FUNCTIONS BY EXTRA FUNCTIONS WHICH SEPERATES THEM INTO SMALLER ONES
+        '''
+        :param name_path_tupl: (name, path) to save the pic
+        :return: a showing of the undirected graph/ saves the picture
+        '''
+        # Setting
+        plt.axis("on")
+        ax = plt.gca()
+        self.create_edgeless_graph_plot(ax)
+        # Undirected labels
+        elarge = [(u, v) for (u, v, d) in self.graph.edges(data=True) if d["weight"]]
+        nx.draw_networkx_edges(self.graph, self.node_pos_dict, edgelist=elarge, width=EDGE_WIDTH, ax=ax)
+
+        # Create Figure
+        self.create_graph_settings_with_cartesic_coord(plt, ax)
+        # Save
+        if name_path_tupl:
+            self.save_fig(name=name_path_tupl[0], path=name_path_tupl[1], format=SAVE_FORMAT, dpi=DPI)
+        # plt.gca().set_aspect('equal', adjustable='box')
+        plt.show()
+
+    def directed_graph_with_path_plt_(self, name_path_tupl=''):
+        '''
+        :param name_path_tupl: (name, path) to save the pic
+        :return: a showing of the undirected graph/ saves the picture
+        '''
+        # Setting
+        plt.axis("on")
+        ax = plt.gca()
+        self.create_edgeless_graph_plot(ax)
+        # Directed labels
+        elarge = [(u, v) for (u, v, d) in self.graph.edges(data=True) if d["weight"]]
+        nx.draw_networkx_edges(self.graph, self.node_pos_dict, edgelist=elarge, width=EDGE_WIDTH, ax=ax,
+                               connectionstyle=CONNECTION_STYLE, arrowsize=ARROW_SIZE)
+        edge_labels = nx.get_edge_attributes(self.graph, 'weight')
+        nx.draw_networkx_edge_labels(self.graph, self.node_pos_dict, edge_labels=edge_labels, ax=ax, label_pos=LABLE_POS,
+                                     font_size=FONT_SIZE_EDGES)
+        # Create figure:
+        self.create_graph_settings_with_cartesic_coord(plt, ax)
+        # Save
+        if name_path_tupl:
+            self.save_fig(name=name_path_tupl[0], path=name_path_tupl[1], format=SAVE_FORMAT, dpi=DPI)
+        plt.gca().set_aspect('equal', adjustable='box')
+        plt.show()
+
+
+    def save_fig(self, name, path: Union[bool, str] = False, format='png', dpi=1200):
+        if path:
+            plt.savefig(path + r'\{}_{}.'.format(name, time.strftime("%Y-%m-%d_H%H-M%M")) + format, format=format, dpi=dpi)
+        else:
+            plt.savefig('{}_{}'.format(name, time.strftime("%Y-%m-%d_H%H-M%M")) + format, format=format, dpi=dpi)
+
+    def create_edgeless_graph_plot(self, ax):
+        for node_type in self.node_type_dict.keys():
+            nlist = self.node_type_dict[node_type]
+            ncolor = self.node_color_dict[node_type]
+
+            # draw the graph
+            nx.draw_networkx_nodes(self.graph,
+                                   pos=self.node_pos_dict,
+                                   nodelist=nlist,
+                                   ax=ax,
+                                   node_color=ncolor,
+                                   label=node_type,
+                                   edgecolors=NODE_EDGE_COLOR,
+                                   node_size=NODE_SIZE)
+
+        nx.draw_networkx_labels(self.graph, self.node_pos_dict, font_size=FONT_SIZE, font_family=FONT_FAMILY)
+
+    def create_graph_settings_with_cartesic_coord(self, plt, ax):
+        # getting the position of the legend so that the graph is not disrupted:
+        x_lim_max = max([x[0] for x in self.node_pos_dict.values()])
+        x_lim_max += x_lim_max * 0.01
+        y_lim_max = max([y[1] for y in self.node_pos_dict.values()])
+        y_lim_max += y_lim_max * 0.05
+        x_lim_min = min([x[0] for x in self.node_pos_dict.values()])
+        x_lim_min -= x_lim_min * 0.01
+        y_lim_min = min([y[1] for y in self.node_pos_dict.values()])
+        y_lim_min -= y_lim_min * 0.05
+
+        ax.tick_params(left=True, bottom=True, labelleft=True, labelbottom=True)
+        legend_size = 16
+        plt.legend(scatterpoints=1, prop={'size': legend_size})
+        plt.xlim(x_lim_min - 0.5, x_lim_max + legend_size / 6)
+        plt.ylim(y_lim_min - 0.5, y_lim_max + 0.5)
+        plt.xlabel('x')
+        plt.ylabel('y')
+        plt.tight_layout()

tool_lib/print_exc_plus.py

@@ -0,0 +1,321 @@
+import inspect
+import os
+import re
+import sys
+import traceback
+from itertools import islice
+from pickle import PickleError
+from typing import Sized, Dict, Tuple, Type
+
+from types import FrameType
+
+from tool_lib.threading_timer_decorator import exit_after
+
+try:
+    import numpy
+except ImportError:
+    numpy = None
+
+FORMATTING_OPTIONS = {
+    'MAX_LINE_LENGTH': 1024,
+    'SHORT_LINE_THRESHOLD': 128,
+    'MAX_NEWLINES': 20,
+}
+ID = int
+
+
+# noinspection PyPep8Naming
+def name_or_str(X):
+    try:
+        return re.search(r"<class '?(.*?)'?>", str(X))[1]
+    except TypeError:  # if not found
+        return str(X)
+
+
+@exit_after(2)
+def type_string(x):
+    if numpy is not None and isinstance(x, numpy.ndarray):
+        return name_or_str(type(x)) + str(x.shape)
+    elif isinstance(x, Sized):
+        return name_or_str(type(x)) + f'({len(x)})'
+    else:
+        return name_or_str(type(x))
+
+
+@exit_after(2)
+def to_string_with_timeout(x):
+    return str(x)
+
+
+def nth_index(iterable, value, n):
+    matches = (idx for idx, val in enumerate(iterable) if val == value)
+    return next(islice(matches, n - 1, n), None)
+
+
+class DumpingException(Exception):
+    pass
+
+
+class SubclassNotFound(ImportError):
+    pass
+
+
+def subclass_by_name(name: str, base: Type):
+    candidates = [t for t in base.__subclasses__() if t.__name__ == name]
+    if len(candidates) != 1:
+        raise SubclassNotFound()
+    return candidates[0]
+
+
+def dont_import():
+    raise ImportError
+
+
+loaded_custom_picklers = False
+
+
+def load_custom_picklers():
+    global loaded_custom_picklers
+    if loaded_custom_picklers:
+        return
+    print('Loading custom picklers...')
+
+    typing_types = ['Dict', 'List', 'Set', 'Tuple', 'Callable', 'Optional']
+    for unpicklable_type in [
+        'from zmq import Socket as unpicklable',
+        'from zmq import Context as unpicklable',
+
+        'from sqlite3 import Connection as unpicklable',
+        'from sqlite3 import Cursor as unpicklable',
+
+        'from socket import socket as unpicklable',
+
+        'from tensorflow import Tensor as unpicklable',
+        'from tensorflow.python.types.core import Tensor as unpicklable',
+        # 'from tensorflow.keras import Model as unpicklable',
+        'from tensorflow.python.eager.def_function import Function as unpicklable',
+        'from tensorflow.python.keras.utils.object_identity import _ObjectIdentityWrapper as unpicklable',
+        # Next line: pybind11_builtins.pybind11_object
+        'from tensorflow.python._pywrap_tfe import TFE_MonitoringBoolGauge0;unpicklable=TFE_MonitoringBoolGauge0.__base__',
+
+        'unpicklable = subclass_by_name(\\"PyCapsule\\", object)',
+        'unpicklable = subclass_by_name(\\"_CData\\", object)',
+
+        'from h5py import HLObject as unpicklable',
+
+        'from builtins import memoryview as unpicklable',
+
+        # can't pickle type annotations from typing in python <= 3.6 (Next line: typing._TypingBase)
+        'import sys;dont_import() if sys.version >=\\"3.7\\" else None;from typing import Optional;unpicklable = type(Optional).__base__.__base__',
+        *[f'import sys;dont_import() if sys.version >=\\"3.7\\" else None;from typing import {t};unpicklable = type({t})' for t in typing_types],
+        'import sys;dont_import() if sys.version >=\\"3.7\\" else None;from typing import Dict;unpicklable = type(Dict).__base__',
+
+        'import inspect;unpicklable = type(inspect.stack()[0].frame)',
+
+        # can't pickle thread-local data
+        'from threading import local as unpicklable',
+
+        # can't pickle generator objects
+        'unpicklable = type(_ for _ in [])',
+    ]:
+        try:
+            unpicklable = eval(f'exec("{unpicklable_type}") or unpicklable')
+        except ImportError:
+            pass
+        except TypeError as e :
+            if 'Descriptors cannot not be created directly' in str(e):
+                pass
+            else:
+                raise
+        else:
+            register_unpicklable(unpicklable, also_subclasses=True)
+        finally:
+            unpicklable = None
+    loaded_custom_picklers = True
+
+
+def register_unpicklable(unpicklable: Type, also_subclasses=False):
+    import dill
+    @dill.register(unpicklable)
+    def save_unpicklable(pickler, obj):
+        def recreate_unpicklable():
+            return f'This was something that could not be pickled and instead was replaced with this string'
+
+        recreate_unpicklable.__name__ = f'unpicklable_{unpicklable.__name__}'
+        pickler.save_reduce(recreate_unpicklable, (), obj=obj)
+
+    if also_subclasses:
+        if unpicklable.__subclasses__ is type.__subclasses__:
+            subclasses = []
+        else:
+            subclasses = unpicklable.__subclasses__()
+        for subclass in subclasses:
+            register_unpicklable(subclass, also_subclasses=True)
+
+
+def dump_stack_to_file(serialize_to, print=print, stack=None):
+    if stack is None:
+        stack = inspect.stack()[1:]
+    try:
+        import dill
+    except ModuleNotFoundError:
+        print('Dill not available. Not dumping stack.')
+    else:
+        print('Dumping stack...')
+        load_custom_picklers()
+
+        serializable_stack = []
+        for frame in stack:
+            if isinstance(frame, inspect.FrameInfo):
+                frame = frame.frame
+            serializable_stack.append({
+                k: frame.__getattribute__(k)
+                for k in ['f_globals', 'f_locals', 'f_code', 'f_lineno', 'f_lasti']
+            })
+
+        with open(serialize_to, 'wb') as serialize_to_file:
+            try:
+                print(f'Dumping stack...')
+                dill.dump(serializable_stack, serialize_to_file)
+            except (PickleError, RecursionError) as e:
+                print(f'Was not able to dump the stack. Error {type(e)}: {e}')
+                unpicklable_frames = []
+                for frame_idx in range(len(serializable_stack)):
+                    try:
+                        dill.dumps(serializable_stack[frame_idx])
+                    except (PickleError, RecursionError):
+                        unpicklable_frames.append(frame_idx)
+                print(f'Unpicklable frames (top=0, bottom={len(serializable_stack)}):', unpicklable_frames)
+
+                if 'typing.' in str(e):
+                    print('This might be fixed by upgrading to python 3.7 or above.')
+            else:
+                print(f'Dumped stack. Can be loaded with:')
+                print(f'with open(r"{os.path.abspath(serialize_to)}", "rb") as f: import dill; dill._dill._reverse_typemap["ClassType"] = type; stack = dill.load(f)')
+        if os.path.isfile(serialize_to):
+            from lib.util import backup_file
+            backup_file(serialize_to)
+
+
+def print_exc_plus(print=print, serialize_to=None, print_trace=True):
+    """
+    Print the usual traceback information, followed by a listing of all the
+    local variables in each frame.
+    """
+    limit = FORMATTING_OPTIONS['MAX_LINE_LENGTH']
+    max_newlines = FORMATTING_OPTIONS['MAX_NEWLINES']
+    tb = sys.exc_info()[2]
+    if numpy is not None:
+        options = numpy.get_printoptions()
+        numpy.set_printoptions(precision=3, edgeitems=2, floatmode='maxprec', threshold=20, linewidth=120)
+    else:
+        options = {}
+    stack = []
+    long_printed_objs: Dict[ID, Tuple[str, FrameType]] = {}
+
+    while tb:
+        stack.append(tb.tb_frame)
+        tb = tb.tb_next
+    if print_trace:
+        for frame in stack:
+            if frame is not stack[0]:
+                print('-' * 40)
+            try:
+                print("Frame %s in %s at line %s" % (frame.f_code.co_name,
+                                                     os.path.relpath(frame.f_code.co_filename),
+                                                     frame.f_lineno))
+            except ValueError:  # if path is not relative
+                print("Frame %s in %s at line %s" % (frame.f_code.co_name,
+                                                     frame.f_code.co_filename,
+                                                     frame.f_lineno))
+            for key, value in frame.f_locals.items():
+                # We have to be careful not to cause a new error in our error
+                # printer! Calling str() on an unknown object could cause an
+                # error we don't want.
+
+                # noinspection PyBroadException
+                try:
+                    key_string = to_string_with_timeout(key)
+                except KeyboardInterrupt:
+                    key_string = "<TIMEOUT WHILE PRINTING KEY>"
+                except Exception:
+                    key_string = "<ERROR WHILE PRINTING KEY>"
+
+                # noinspection PyBroadException
+                try:
+                    type_as_string = type_string(value)
+                except KeyboardInterrupt:
+                    type_as_string = "<TIMEOUT WHILE PRINTING TYPE>"
+                except Exception as e:
+                    # noinspection PyBroadException
+                    try:
+                        type_as_string = f"<{type(e).__name__} WHILE PRINTING TYPE>"
+                    except Exception:
+                        type_as_string = "<ERROR WHILE PRINTING TYPE>"
+
+                if id(value) in long_printed_objs:
+                    prev_key_string, prev_frame = long_printed_objs[id(value)]
+                    if prev_frame is frame:
+                        print("\t%s is the same as '%s'" %
+                              (key_string + ' : ' + type_as_string,
+                               prev_key_string))
+                    else:
+                        print("\t%s is the same as '%s' in frame %s in %s at line %s." %
+                              (key_string + ' : ' + type_as_string,
+                               prev_key_string,
+                               prev_frame.f_code.co_name,
+                               os.path.relpath(prev_frame.f_code.co_filename),
+                               prev_frame.f_lineno))
+                    continue
+
+                # noinspection PyBroadException
+                try:
+                    value_string = to_string_with_timeout(value)
+                except KeyboardInterrupt:
+                    value_string = "<TIMEOUT WHILE PRINTING VALUE>"
+                except Exception:
+                    value_string = "<ERROR WHILE PRINTING VALUE>"
+                line: str = '\t' + key_string + ' : ' + type_as_string + ' = ' + value_string
+                if limit is not None and len(line) > limit:
+                    line = line[:limit - 1] + '...'
+                if max_newlines is not None and line.count('\n') > max_newlines:
+                    line = line[:nth_index(line, '\n', max_newlines)].strip() + '... (' + str(
+                        line[nth_index(line, '\n', max_newlines):].count('\n')) + ' more lines)'
+                if len(line) > FORMATTING_OPTIONS['SHORT_LINE_THRESHOLD']:
+                    long_printed_objs[id(value)] = key_string, frame
+                print(line)
+
+        traceback.print_exc()
+
+        etype, value, tb = sys.exc_info()
+        for line in traceback.TracebackException(type(value), value, tb, limit=limit).format(chain=True):
+            print(line)
+
+    if serialize_to is not None:
+        dump_stack_to_file(stack=stack, serialize_to=serialize_to, print=print)
+
+    if numpy is not None:
+        numpy.set_printoptions(**options)
+
+
+def main():
+    def fun1(c, d, e):
+        return fun2(c, d + e)
+
+    def fun2(g, h):
+        raise RuntimeError
+
+    def fun3(z):
+        return numpy.zeros(shape=z)
+
+    try:
+        import numpy as np
+        fun1(numpy.random.normal(size=(3, 4, 5, 6)), '12321', '123')
+        data = '???' * 100
+        fun3(data)
+    except:
+        print_exc_plus()
+
+
+if __name__ == '__main__':
+    main()

tool_lib/progress_bar.py

@@ -0,0 +1,194 @@
+import functools
+import math
+import time
+from math import floor
+from typing import Iterable, Sized, Iterator
+
+
+class ProgressBar(Sized, Iterable):
+    def __iter__(self) -> Iterator:
+        self.check_if_num_steps_defined()
+        self.current_iteration = -1  # start counting at the end of the first epoch
+        self.current_iterator = iter(self._backend)
+        self.start_time = time.perf_counter()
+        return self
+
+    def __init__(self,
+                 num_steps=None,
+                 prefix='',
+                 suffix='',
+                 line_length=75,
+                 empty_char='-',
+                 fill_char='#',
+                 print_eta=True,
+                 decimals=1):
+        self.decimals = decimals
+        self.line_length = line_length
+        self.suffix = suffix
+        self.empty_char = empty_char
+        self.prefix = prefix
+        self.fill_char = fill_char
+        self.print_eta = print_eta
+        self.current_iteration = 0
+        self.last_printed_value = None
+        self.current_iterator = None
+        self.start_time = time.perf_counter()
+
+        try:
+            self._backend = range(num_steps)
+        except TypeError:
+            if isinstance(num_steps, Sized):
+                if isinstance(num_steps, Iterable):
+                    self._backend = num_steps
+                else:
+                    self._backend = range(len(num_steps))
+            elif num_steps is None:
+                self._backend = None
+            else:
+                raise
+
+        assert num_steps is None or isinstance(self._backend, (Iterable, Sized))
+
+    def set_num_steps(self, num_steps):
+        try:
+            self._backend = range(num_steps)
+        except TypeError:
+            if isinstance(num_steps, Sized):
+                if isinstance(num_steps, Iterable):
+                    self._backend = num_steps
+                else:
+                    self._backend = range(len(num_steps))
+            elif num_steps is None:
+                self._backend = None
+            else:
+                raise
+
+        assert num_steps is None or isinstance(self._backend, (Iterable, Sized))
+
+    def __len__(self):
+        return len(self._backend)
+
+    def __next__(self):
+        self.print_progress()
+        try:
+            result = next(self.current_iterator)
+            self.increment_iteration()
+            self.print_progress()
+            return result
+        except StopIteration:
+            self.increment_iteration()
+            self.print_progress()
+            raise
+
+    def step(self, num_iterations=1):
+        self.current_iteration += num_iterations
+        self.print_progress()
+
+    def print_progress(self, iteration=None):
+        """
+        Call in a loop to create terminal progress bar
+        @params:
+            iteration   - Optional  : current iteration (Int)
+        """
+        if iteration is not None:
+            self.current_iteration = iteration
+        try:
+            progress = self.current_iteration / len(self)
+        except ZeroDivisionError:
+            progress = 1
+        if self.current_iteration == 0:
+            self.start_time = time.perf_counter()
+        if self.print_eta and progress > 0:
+            time_spent = (time.perf_counter() - self.start_time)
+            eta = time_spent / progress * (1 - progress)
+            if progress == 1:
+                eta = f' T = {int(time_spent / 60):02d}:{round(time_spent % 60):02d}'
+            else:
+                eta = f' ETA {int(eta / 60):02d}:{round(eta % 60):02d}'
+        else:
+            eta = ''
+        percent = ("{0:" + str(4 + self.decimals) + "." + str(self.decimals) + "f}").format(100 * progress)
+        bar_length = self.line_length - len(self.prefix) - len(self.suffix) - len(eta) - 4 - 6
+        try:
+            filled_length = int(bar_length * self.current_iteration // len(self))
+        except ZeroDivisionError:
+            filled_length = bar_length
+        if math.isclose(bar_length * progress, filled_length):
+            overflow = 0
+        else:
+            overflow = bar_length * progress - filled_length
+            overflow *= 10
+            overflow = floor(overflow)
+        assert overflow in range(10), overflow
+        if overflow > 0:
+            bar = self.fill_char * filled_length + str(overflow) + self.empty_char * (bar_length - filled_length - 1)
+        else:
+            bar = self.fill_char * filled_length + self.empty_char * (bar_length - filled_length)
+
+        print_value = '\r{0} |{1}| {2}% {4}{3}'.format(self.prefix, bar, percent, eta, self.suffix)
+        if self.current_iteration == len(self):
+            print_value += '\n'  # Print New Line on Complete
+        if self.last_printed_value == print_value:
+            return
+        self.last_printed_value = print_value
+        print(print_value, end='')
+
+    def increment_iteration(self):
+        self.current_iteration += 1
+        if self.current_iteration > len(self):  # catches the special case at the end of the bar
+            self.current_iteration %= len(self)
+
+    def monitor(self, func=None):
+        """ Decorates the given function func to print a progress bar before and after each call. """
+        if func is None:
+            # Partial application, to be able to specify extra keyword
+            # arguments in decorators
+            return functools.partial(self.monitor)
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            self.check_if_num_steps_defined()
+            self.print_progress()
+            result = func(*args, **kwargs)
+            self.increment_iteration()
+            self.print_progress()
+            return result
+
+        return wrapper
+
+    def check_if_num_steps_defined(self):
+        if self._backend is None:
+            raise RuntimeError('You need to specify the number of iterations before starting to iterate. '
+                               'You can either pass it to the constructor or use the method `set_num_steps`.')
+
+
+if __name__ == '__main__':
+    # Einfach beim iterieren verwenden
+    for x in ProgressBar([0.5, 2, 0.5]):
+        time.sleep(x)
+
+    # manuell aufrufen
+    data = [1, 5, 5, 6, 12, 3, 4, 5]
+    y = 0
+    p = ProgressBar(len(data))
+    for x in data:
+        p.print_progress()
+        time.sleep(0.2)
+        y += x
+        p.current_iteration += 1
+        p.print_progress()
+
+    print(y)
+
+    # oder einfach bei jedem funktionsaufruf den balken printen
+    p = ProgressBar()
+
+
+    @p.monitor
+    def heavy_computation(t=0.25):
+        time.sleep(t)
+
+
+    p.set_num_steps(10)  # 10 steps pro balken
+    for _ in range(20):  # zeichnet 2 balken
+        heavy_computation(0.25)

tool_lib/python_gui.py

@@ -0,0 +1,134 @@
+import PySimpleGUI as sg
+import os.path
+
+
+# First the window layout in 2 columns
+
+
+
+def optiwae_gui(obj):
+    file_list_column = [
+
+        [
+
+            sg.Text("Type Section"),
+
+            sg.In(size=(25, 1), enable_events=True, key="-FOLDER-"),
+
+            sg.FolderBrowse(),
+
+        ],
+
+        [
+
+            sg.Listbox(
+
+                values=[], enable_events=True, size=(40, 20), key="-FILE LIST-"
+
+            )
+
+        ],
+
+    ]
+    variables_viewer_column = [
+        [sg.Text('x')],
+
+
+    ]
+
+    # ----- Full layout -----
+    lines_of_variables_0 = []
+    lines_of_variables_1 = []
+    lines_of_variables_2 = []
+    for count,(key, value) in enumerate(obj.items()):
+        if count<= 10:
+            lines_of_variables_0 += [[sg.Text(str(key) + '\n')]]
+        if count >10:
+            lines_of_variables_1+= [[sg.Text(str(key) +'\n')]]
+        if count >20:
+            lines_of_variables_2+= [[sg.Text(str(key) +'\n')]]
+
+
+    layout = [
+
+            # sg.Column(file_list_column),
+            [sg.Text('Enter Changes', size=(15, 1)), sg.InputText()],
+            [sg.Submit(), sg.Cancel()],
+
+
+            sg.Frame(layout= lines_of_variables_0, title=''),
+            sg.VSeperator(),
+            sg.Frame(layout=lines_of_variables_1, title=''),
+            sg.VSeperator(),
+            sg.Frame(layout=lines_of_variables_2, title=''),
+
+
+        ]
+
+
+    layout = [layout]
+    window = sg.Window("Variables Viewer", layout)
+    list_of_commands = []
+    while True:
+
+        event, values = window.read()
+        list_of_commands += [values[0]]
+
+        if event == "Exit" or event == sg.WIN_CLOSED or event=='Cancel':
+            window.close()
+            break
+
+
+    return list(set(list_of_commands))
+
+
+
+    # if event == "-FOLDER-":
+    #
+    #     folder = values["-FOLDER-"]
+    #
+    #     try:
+    #
+    #         # Get list of files in folder
+    #
+    #         file_list = os.listdir(folder)
+    #
+    #     except:
+    #
+    #         file_list = []
+    #
+    #     fnames = [
+    #
+    #         f
+    #
+    #         for f in file_list
+    #
+    #         if os.path.isfile(os.path.join(folder, f))
+    #
+    #            and f.lower().endswith((".png", ".gif"))
+    #
+    #     ]
+    #
+    #     window["-FILE LIST-"].update(fnames)
+    # elif event == "-FILE LIST-":  # A file was chosen from the listbox
+    #
+    #     try:
+    #
+    #         filename = os.path.join(
+    #
+    #             values["-FOLDER-"], values["-FILE LIST-"][0]
+    #
+    #         )
+    #
+    #         window["-TOUT-"].update(filename)
+    #
+    #         window["-IMAGE-"].update(filename=filename)
+    #
+    #     except:
+    #
+    #         pass
+
+
+if __name__ == '__main__':
+    k= {'x' :5, 'y':6}
+    optiwae_gui(k)

tool_lib/python_to_latex.py

@@ -0,0 +1,38 @@
+'''This file is created to be an Interface between Python and Latex.
+   Goals are: Creating Latexfigures out of Plots '''
+from pylatex import Document,Section,Subsection,Tabular,Math, TikZ, Axis, Plot, Figure, Matrix, Alignat
+import numpy as np
+import os
+
+class PythonLatex:
+
+    def __init__(self, geometrie_options, file_path, data=[]):
+        self.geometrie_options = geometrie_options
+        self.file_path = file_path
+        self.data = data
+        self.doc = Document(geometry_options= self.geometrie_options)
+
+    def get_table(self):
+        pass
+
+    def get_equation(self):
+        pass
+
+    def get_matrix(self):
+        pass
+
+
+class LatexPlots(PythonLatex):
+
+    def __init__(self, plot_options, image_file_path=''):
+        super().__init__(geometrie_options=geometrie_options)
+        self.plot_options = plot_options
+        if image_file_path:
+            self.image_filename = os.path.join(os.path.dirname(__file__), image_file_path)
+        else:
+            self.image_filename = ''
+
+
+if __name__ == '__main__':
+    geometrie_options = {'tmargin': '1cm', 'lmargin': '10cm'}
+    PythonLatex(geometrie_options=geometrie_options)

tool_lib/random_generator.py

@@ -0,0 +1,19 @@
+import random
+
+# !!!!!!!!!!!!!!!!! NOT UPDATED !!!!!!!!!!!!!!!!!!!!!
+# def get_random_dict_elements(dict_, count):
+#     random_dict = {}
+#     for i in range(0, count):
+#         key, value = random.choice(list(dict_.items()))
+#         random_dict[key] = value
+#     return random_dict
+
+
+def get_random_list_elements(list_, count):
+    list_2 = list_.copy()
+    random_list = []
+    for i in range(0, min(count,len(list_2))):
+        elem = random.choice(list_2)
+        random_list += [elem]
+        list_2.remove(elem)
+    return random_list

tool_lib/run_multiple_jobs.py

@@ -0,0 +1,32 @@
+#################################################################################
+#Import (here could be project specific imports, clean after use))
+import inspect
+#################################################################################
+
+def run_multiple_jobs_with_constants(ordered_list_of_functions,dict_of_functions,dict_constants=[]):
+    '''
+    This function is made to run through multiple functions in different files,
+    these functions can contain constants which can be given as additional Input
+    :param ordered_list_of_functions: a list in which the function handles are in
+    :param dict_of_functions: a dictionary which maps the
+    :param dict_constants: constants which are used for the functions
+    '''
+
+    pass
+
+def run_multiple_jobs_with_alternating_constants(ordered_list_of_functions,dict_constants):
+    pass
+
+def run_single_job_multiple_times(ordered_list_of_functions,dict_constants):
+    pass
+
+def run_single_job_multiple_times_alternation():
+    pass
+
+#################################################################################
+#this function can be used if u want to run this file
+def main():
+    pass
+
+if __name__ == '__main__':  # only if we execute this file but not if we import it
+    main()

tool_lib/stack_tracer.py

@@ -0,0 +1,105 @@
+"""Stack tracer for multi-threaded applications.
+
+
+Usage:
+
+import stacktracer
+stacktracer.start_trace("trace.html",interval=5,auto=True) # Set auto flag to always update file!
+....
+stacktracer.stop_trace()
+"""
+
+import sys
+import traceback
+from pygments import highlight
+from pygments.lexers import PythonLexer
+from pygments.formatters import HtmlFormatter
+
+
+# Taken from http://bzimmer.ziclix.com/2008/12/17/python-thread-dumps/
+
+def stacktraces():
+    code = []
+    for threadId, stack in sys._current_frames().items():
+        code.append("\n# ThreadID: %s" % threadId)
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
+            if line:
+                code.append("  %s" % (line.strip()))
+
+    return highlight("\n".join(code), PythonLexer(), HtmlFormatter(
+        full=False,
+        # style="native",
+        noclasses=True,
+    ))
+
+
+# This part was made by nagylzs
+import os
+import time
+import threading
+
+
+class TraceDumper(threading.Thread):
+    """Dump stack traces into a given file periodically."""
+
+    def __init__(self, fpath, interval, auto):
+        """
+        @param fpath: File path to output HTML (stack trace file)
+        @param auto: Set flag (True) to update trace continuously.
+            Clear flag (False) to update only if file not exists.
+            (Then delete the file to force update.)
+        @param interval: In seconds: how often to update the trace file.
+        """
+        assert (interval > 0.1)
+        self.auto = auto
+        self.interval = interval
+        self.fpath = os.path.abspath(fpath)
+        self.stop_requested = threading.Event()
+        threading.Thread.__init__(self)
+
+    def run(self):
+        while not self.stop_requested.isSet():
+            time.sleep(self.interval)
+            if self.auto or not os.path.isfile(self.fpath):
+                self.stacktraces()
+
+    def stop(self):
+        self.stop_requested.set()
+        self.join()
+        try:
+            if os.path.isfile(self.fpath):
+                os.unlink(self.fpath)
+        except:
+            pass
+
+    def stacktraces(self):
+        fout = open(self.fpath, "w+")
+        try:
+            fout.write(stacktraces())
+        finally:
+            fout.close()
+
+
+_tracer = None
+
+
+def trace_start(fpath, interval=5, auto=True):
+    """Start tracing into the given file."""
+    global _tracer
+    if _tracer is None:
+        _tracer = TraceDumper(fpath, interval, auto)
+        _tracer.setDaemon(True)
+        _tracer.start()
+    else:
+        raise Exception("Already tracing to %s" % _tracer.fpath)
+
+
+def trace_stop():
+    """Stop tracing."""
+    global _tracer
+    if _tracer is None:
+        raise Exception("Not tracing, cannot stop.")
+    else:
+        _tracer.stop()
+        _tracer = None

tool_lib/threading_timer_decorator.py

@@ -0,0 +1,116 @@
+from __future__ import print_function
+
+import sys
+import threading
+from functools import partial
+from time import sleep
+
+try:
+    import thread
+except ImportError:
+    import _thread as thread
+
+try:  # use code that works the same in Python 2 and 3
+    range, _print = xrange, print
+
+
+    def print(*args, **kwargs):
+        flush = kwargs.pop('flush', False)
+        _print(*args, **kwargs)
+        if flush:
+            kwargs.get('file', sys.stdout).flush()
+except NameError:
+    pass
+
+
+class Cancellation:
+    def __init__(self, canceled=False):
+        self.canceled = canceled
+
+
+def cdquit(fn_name, cancellation):
+    if cancellation.canceled:
+        return
+    # print to stderr, unbuffered in Python 2.
+    print('{0} took too long'.format(fn_name), file=sys.stderr)
+    sys.stderr.flush()  # Python 3 stderr is likely buffered.
+    thread.interrupt_main()  # raises KeyboardInterrupt
+
+
+def exit_after(s):
+    '''
+    use as decorator to exit process if
+    function takes longer than s seconds
+    '''
+
+    def outer(fn):
+        def inner(*args, **kwargs):
+            c = Cancellation()
+            timer = threading.Timer(s, partial(cdquit, cancellation=c), args=[fn.__name__])
+            timer.start()
+            try:
+                result = fn(*args, **kwargs)
+            finally:
+                c.canceled = True
+                timer.cancel()
+            return result
+
+        return inner
+
+    return outer
+
+
+def call_method_with_timeout(method, timeout, *args, **kwargs):
+    return exit_after(timeout)(method)(*args, **kwargs)
+
+
+@exit_after(1)
+def a():
+    print('a')
+
+
+@exit_after(2)
+def b():
+    print('b')
+    sleep(1)
+
+
+@exit_after(3)
+def c():
+    print('c')
+    sleep(2)
+
+
+@exit_after(4)
+def d():
+    print('d started')
+    for i in range(10):
+        sleep(1)
+        print(i)
+
+
+@exit_after(5)
+def countdown(n):
+    print('countdown started', flush=True)
+    for i in range(n, -1, -1):
+        print(i, end=', ', flush=True)
+        sleep(1)
+    print('countdown finished')
+
+
+def main():
+    a()
+    b()
+    c()
+    try:
+        d()
+    except KeyboardInterrupt as error:
+        print('d should not have finished, printing error as expected:')
+        print(error)
+    countdown(3)
+    countdown(10)
+    print('This should not print!!!')
+
+
+if __name__ == '__main__':
+    main()

tool_lib/tuned_cache.py

@@ -0,0 +1,130 @@
+import functools
+import sys
+from copy import deepcopy
+
+assert 'joblib' not in sys.modules, 'Import tuned cache before joblib'
+
+# noinspection PyProtectedMember,PyPep8
+import joblib
+# noinspection PyProtectedMember,PyPep8
+from joblib.func_inspect import _clean_win_chars
+# noinspection PyProtectedMember,PyPep8
+from joblib.memory import MemorizedFunc, _FUNCTION_HASHES, NotMemorizedFunc, Memory
+
+_FUNC_NAMES = {}
+
+
+# noinspection SpellCheckingInspection
+class TunedMemory(Memory):
+    def cache(self, func=None, ignore=None, verbose=None, mmap_mode=False):
+        """ Decorates the given function func to only compute its return
+            value for input arguments not cached on disk.
+
+            Parameters
+            ----------
+            func: callable, optional
+                The function to be decorated
+            ignore: list of strings
+                A list of arguments name to ignore in the hashing
+            verbose: integer, optional
+                The verbosity mode of the function. By default that
+                of the memory object is used.
+            mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
+                The memmapping mode used when loading from cache
+                numpy arrays. See numpy.load for the meaning of the
+                arguments. By default that of the memory object is used.
+
+            Returns
+            -------
+            decorated_func: MemorizedFunc object
+                The returned object is a MemorizedFunc object, that is
+                callable (behaves like a function), but offers extra
+                methods for cache lookup and management. See the
+                documentation for :class:`joblib.memory.MemorizedFunc`.
+        """
+        if func is None:
+            # Partial application, to be able to specify extra keyword
+            # arguments in decorators
+            return functools.partial(self.cache, ignore=ignore,
+                                     verbose=verbose, mmap_mode=mmap_mode)
+        if self.store_backend is None:
+            return NotMemorizedFunc(func)
+        if verbose is None:
+            verbose = self._verbose
+        if mmap_mode is False:
+            mmap_mode = self.mmap_mode
+        if isinstance(func, TunedMemorizedFunc):
+            func = func.func
+        return TunedMemorizedFunc(func, location=self.store_backend,
+                                  backend=self.backend,
+                                  ignore=ignore, mmap_mode=mmap_mode,
+                                  compress=self.compress,
+                                  verbose=verbose, timestamp=self.timestamp)
+
+
+class TunedMemorizedFunc(MemorizedFunc):
+    def __call__(self, *args, **kwargs):
+        # Also store in the in-memory store of function hashes
+        if self.func not in _FUNCTION_HASHES:
+            is_named_callable = (hasattr(self.func, '__name__') and
+                                 self.func.__name__ != '<lambda>')
+            if is_named_callable:
+                # Don't do this for lambda functions or strange callable
+                # objects, as it ends up being too fragile
+                func_hash = self._hash_func()
+                try:
+                    _FUNCTION_HASHES[self.func] = func_hash
+                except TypeError:
+                    # Some callable are not hashable
+                    pass
+
+        # return same result as before
+        return MemorizedFunc.__call__(self, *args, **kwargs)
+
+
+old_get_func_name = joblib.func_inspect.get_func_name
+
+
+def tuned_get_func_name(func, resolv_alias=True, win_characters=True):
+    if (func, resolv_alias, win_characters) not in _FUNC_NAMES:
+        _FUNC_NAMES[(func, resolv_alias, win_characters)] = old_get_func_name(func, resolv_alias, win_characters)
+
+        if len(_FUNC_NAMES) > 1000:
+            # keep cache small and fast
+            for idx, k in enumerate(_FUNC_NAMES.keys()):
+                if idx % 2:
+                    del _FUNC_NAMES[k]
+        # print('cache size ', len(_FUNC_NAMES))
+
+    return deepcopy(_FUNC_NAMES[(func, resolv_alias, win_characters)])
+
+
+joblib.func_inspect.get_func_name = tuned_get_func_name
+joblib.memory.get_func_name = tuned_get_func_name
+
+
+def main():
+    class A:
+        test_cache = TunedMemory('.cache/test_cache', verbose=1)
+
+        def __init__(self, a):
+            self.a = a
+            self.compute = self.test_cache.cache(self.compute)
+
+        def compute(self):
+            return self.a + 1
+
+    a1, a2 = A(2), A(2)
+    print(a1.compute())
+    print('---')
+    print(a2.compute())
+    print('---')
+    a1.a = 3
+    print(a1.compute())
+    print('---')
+    print(a2.compute())
+    print('---')
+
+
+if __name__ == '__main__':
+    main()

tool_lib/unlock.py

@@ -0,0 +1,35 @@
+import logging
+import threading
+from typing import Union
+
+import gevent.threading
+
+
+class UnLock(object):
+    """
+    The idea is to use this class within a context manager to release a lock temporarily but then acquire it again.
+    Example:
+        lock = gevent.threading.Lock()
+        with lock:
+            logging.info('Websocket request 10 received')
+            with UnLock(lock):
+                sleep(10.)  # this part does not need the resource to be locked
+            logging.info('Websocket request 10 finished')
+    """
+
+    def __init__(self, lock: Union[gevent.threading.Lock, threading.Lock], require_taken: bool = True, verbose=False):
+        self.lock = lock
+        self.require_taken = require_taken
+        self.verbose = verbose
+
+    def __enter__(self):
+        if self.require_taken and not self.lock.locked():
+            raise RuntimeError('Lock not taken')
+        self.lock.release()
+        if self.verbose:
+            logging.info('Unlocked')
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.lock.acquire()
+        if self.verbose:
+            logging.info('Locked again.')

tool_lib/util.py

@@ -12,15 +12,17 @@ import sqlite3
 import sys
 import threading
 import time
+import typing
 from bisect import bisect_left
 from enum import Enum
 from itertools import chain, combinations
 from math import log, isnan, nan, floor, log10, gcd
 from numbers import Number
 from shutil import copyfile
+from subprocess import check_output, CalledProcessError, PIPE
 from threading import RLock
 from types import FunctionType
-from typing import Union, Tuple, List, Optional, Dict, Any, Type
+from typing import Union, Tuple, List, Optional, Dict, Any, Type, Callable, Literal
 # noinspection PyUnresolvedReferences
 from unittest import TestCase, mock
 
@@ -36,6 +38,9 @@ import scipy.stats
 import tabulate
 from scipy.ndimage import zoom
 
+from tool_lib import stack_tracer, print_exc_plus
+from tool_lib.my_logger import logging
+
 X = Y = Z = float
 
 
@@ -46,10 +51,13 @@ class KnownIssue(Exception):
     pass
 
 
-def powerset(iterable):
+def powerset(iterable, largest_first=False):
     """powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"""
     s = list(iterable)
-    return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1))
+    sizes = range(len(s) + 1)
+    if largest_first:
+        sizes = reversed(sizes)
+    return chain.from_iterable(combinations(s, r) for r in sizes)
 
 
 def plot_with_conf(x, y_mean, y_conf, alpha=0.5, **kwargs):
@@ -74,6 +82,10 @@ def choice(sequence, probabilities):
     raise AssertionError('Probabilities must sum to 1')
 
 
+def local_timezone():
+    return datetime.datetime.now(datetime.timezone(datetime.timedelta(0))).astimezone().tzinfo
+
+
 def print_attributes(obj, include_methods=False, ignore=None):
     if ignore is None:
         ignore = []
@@ -93,7 +105,7 @@ def attr_dir(obj, include_methods=False, ignore=None):
     return {attr: obj.__getattr__(attr)
             for attr in dir(obj)
             if not attr.startswith('_') and (
-                include_methods or not callable(obj.__getattr__(attr))) and attr not in ignore}
+                    include_methods or not callable(obj.__getattr__(attr))) and attr not in ignore}
 
 
 def zoom_to_shape(a: np.ndarray, shape: Tuple, mode: str = 'smooth', verbose=1):
@@ -166,20 +178,6 @@ def zoom_to_shape(a: np.ndarray, shape: Tuple, mode: str = 'smooth', verbose=1):
         return NotImplementedError('Mode not available.')
 
 
-def profile_wall_time_instead_if_profiling():
-    try:
-        import yappi
-    except ModuleNotFoundError:
-        return
-    currently_profiling = len(yappi.get_func_stats())
-    if currently_profiling and yappi.get_clock_type() != 'wall':
-        yappi.stop()
-        print('Profiling wall time instead of cpu time.')
-        yappi.clear_stats()
-        yappi.set_clock_type("wall")
-        yappi.start()
-
-
 def dummy_computation(*_args, **_kwargs):
     pass
 
@@ -619,78 +617,6 @@ def lru_cache_by_id(maxsize):
     return cachetools.cached(cachetools.LRUCache(maxsize=maxsize), key=id)
 
 
-class EquivalenceRelation:
-    def equivalent(self, a, b) -> bool:
-        raise NotImplementedError('Abstract method')
-
-    def equivalence_classes(self, xs: list):
-        classes = []
-        for x in xs:
-            for c in classes:
-                if self.equivalent(x, c[0]):
-                    c.append(x)
-                    break
-            else:
-                classes.append([x])
-        return classes
-
-    def check_reflexivity_on_dataset(self, xs):
-        for x in xs:
-            if not self.equivalent(x, x):
-                return False
-        return True
-
-    def check_symmetry_on_dataset(self, xs):
-        for x in xs:
-            for y in xs:
-                if x is y:
-                    continue
-                if self.equivalent(x, y) and not self.equivalent(y, x):
-                    return False
-        return True
-
-    def check_axioms_on_dataset(self, xs):
-        return (
-            self.check_reflexivity_on_dataset(xs)
-            and self.check_symmetry_on_dataset(xs)
-            and self.check_transitivity_on_dataset(xs, assume_symmetry=True, assume_reflexivity=True)
-        )
-
-    def check_transitivity_on_dataset(self, xs, assume_symmetry=False, assume_reflexivity=False):
-        for x_idx, x in enumerate(xs):
-            for y_idx, y in enumerate(xs):
-                if x is y:
-                    continue
-                if self.equivalent(x, y):
-                    for z_idx, z in enumerate(xs):
-                        if y is z:
-                            continue
-                        if assume_symmetry and x_idx > z_idx:
-                            continue
-                        if assume_reflexivity and x is z:
-                            continue
-                        if self.equivalent(y, z):
-                            if not self.equivalent(x, z):
-                                return False
-        return True
-
-    def match_lists(self, xs, ys, filter_minimum_size=0, filter_maximum_size=math.inf):
-        xs = list(xs)
-        ys = list(ys)
-        if any(x is y for x in xs for y in ys):
-            raise ValueError('Lists contain the same element. This is currently not supported.')
-        classes = self.equivalence_classes([*xs, *ys])
-
-        return [
-            [
-                (0 if any(x2 is x for x2 in xs) else 1, x)
-                for x in c
-            ]
-            for c in classes[::-1]
-            if filter_minimum_size <= len(c) <= filter_maximum_size
-        ]
-
-
 def iff_patch(patch: mock._patch):
     def decorator(f):
         def wrapped(*args, **kwargs):
@@ -726,7 +652,6 @@ def iff_not_patch(patch: mock._patch):
 
 
 EMAIL_CRASHES_TO = []
-VOICE_CALL_ON_CRASH: List[Tuple[str, str]] = []
 
 
 def list_logger(base_logging_function, store_in_list: list):
@@ -763,11 +688,7 @@ def main_wrapper(f):
                                           serialize_to='logs/' + os.path.split(__main__.__file__)[-1] + '.dill')
             for recipient in EMAIL_CRASHES_TO:
                 from jobs.sending_emails import send_mail
-                send_mail.create_simple_mail_via_gmail(body='\n'.join(error_messages), filepath=None, excel_name=None, to_mail=recipient, subject='[python] Crash report')
-            for to_number, from_number in VOICE_CALL_ON_CRASH:
-                logging.info(f'Calling {from_number} to notify about the crash.')
-                voice_call('This is a notification message that one of your python scripts has crashed. If you are unsure about the origin of this call, please contact Eren Yilmaz.',
-                           to_number, from_number)
+                send_mail.create_simple_mail_via_gmail(body='\n'.join(error_messages), filepath=None, excel_name=None, to_mail=recipient, subject='Crash report')
         finally:
             logging.info('Terminated.')
             total_time = time.perf_counter() - start
@@ -786,22 +707,6 @@ def main_wrapper(f):
     return wrapper
 
 
-def voice_call(msg, to_number, from_number):
-    from twilio.rest import Client
-    account_sid = 'AC63c459168c3e4fe34e462acb4f44f748'
-    auth_token = 'b633bc0e945fe7cb737fdac395cc71d6'
-    client = Client(account_sid, auth_token)
-
-    call = client.calls.create(
-                            twiml=f'<Response><Say>{msg}</Say></Response>',
-                            from_=from_number,
-                            to=to_number,
-                        )
-
-    print(call.sid)
-
-
-
 def required_size_for_safe_rotation(base: Tuple[X, Y, Z], rotate_range_deg) -> Tuple[X, Y, Z]:
     if abs(rotate_range_deg) > 45:
         raise NotImplementedError
@@ -887,23 +792,14 @@ def get_all_subclasses(klass):
     return all_subclasses
 
 
-def my_mac_address():
-    """
-    https://stackoverflow.com/a/160821
-    """
-    import uuid
-    mac = uuid.getnode()
-    if (mac >> 40) % 2:
-        return None
-    mac = uuid.UUID(int=mac).hex[-12:]
-    return mac
-
-
-
 def latin1_json(data):
     return json.dumps(data, ensure_ascii=False).encode('latin-1')
 
 
+def utf8_json(data):
+    return json.dumps(data, ensure_ascii=False).encode('utf-8')
+
+
 def l2_norm(v1, v2):
     if len(v1) != len(v2):
         raise ValueError('Both vectors must be of the same size')
@@ -920,6 +816,29 @@ def allow_additional_unused_keyword_arguments(func):
     return wrapper
 
 
+class BiDict(dict):
+    """
+    https://stackoverflow.com/a/21894086
+    """
+    def __init__(self, *args, **kwargs):
+        super(BiDict, self).__init__(*args, **kwargs)
+        self.inverse = {}
+        for key, value in self.items():
+            self.inverse.setdefault(value, []).append(key)
+
+    def __setitem__(self, key, value):
+        if key in self:
+            self.inverse[self[key]].remove(key)
+        super(BiDict, self).__setitem__(key, value)
+        self.inverse.setdefault(value, []).append(key)
+
+    def __delitem__(self, key):
+        self.inverse.setdefault(self[key], []).remove(key)
+        if self[key] in self.inverse and not self.inverse[self[key]]:
+            del self.inverse[self[key]]
+        super(BiDict, self).__delitem__(key)
+
+
 def copy_and_rename_method(func, new_name):
     funcdetails = [
         func.__code__,
@@ -1001,36 +920,106 @@ def running_workers(executor):
                if t == 1)
 
 
+class Bunch(dict):
+    def __init__(self, **kwargs):
+        dict.__init__(self, kwargs)
+        self.__dict__.update(kwargs)
+
+    def add_method(self, m):
+        setattr(self, m.__name__, functools.partial(m, self))
+
+
 def queued_calls(executor):
     return len(executor._work_queue.queue)
 
 
-def retry_on_error(max_tries=3, delay=0.5, backoff=2, only_error_classes=Exception):
-    def decorator(func):
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            for i in range(max_tries):
-                try:
-                    return func(*args, **kwargs)
-                except only_error_classes as e:
-                    if i == max_tries - 1:
-                        raise
-                    logging.error(f'Re-try after error in {func.__name__}: {type(e).__name__}, {e}')
-                    time.sleep(delay * (backoff ** i))
-        return wrapper
-    return decorator
+def single_use_function(f: Callable):
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+        if not wrapper.already_called:
+            wrapper.already_called = True
+            return f(*args, **kwargs)
+        else:
+            raise RuntimeError(f'{f} is supposed to be called only once.')
 
+    wrapper.already_called = False
+    return wrapper
+
+
+def single_use_method(f: Callable):
+    @functools.wraps(f)
+    def wrapper(self, *args, **kwargs):
+        if not hasattr(self, f'_already_called_{id(wrapper)}') or not getattr(self, f'_already_called_{id(wrapper)}'):
+            setattr(self, f'_already_called_{id(wrapper)}', True)
+            return f(self, *args, **kwargs)
+        else:
+            raise RuntimeError(f'{f} is supposed to be called only once per object.')
+
+    return wrapper
 
 
-class EBC:
-    SUBCLASSES_BY_NAME: Dict[str, Type['EBC']] = {}
+class JSONConvertible:
+    SUBCLASSES_BY_NAME: Dict[str, Type[typing.Self]] = {}
 
     def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)
-        EBC.SUBCLASSES_BY_NAME[cls.__name__] = cls
+        JSONConvertible.SUBCLASSES_BY_NAME[cls.__name__] = cls
 
+    @classmethod
+    def subclasses(cls, strict=False):
+        return [t for t in cls.SUBCLASSES_BY_NAME.values() if issubclass(t, cls) and (not strict or t != cls)]
+
+    def to_json(self) -> Dict[str, Any]:
+        raise NotImplementedError('Abstract method')
+
+    @staticmethod
+    def from_json(data: Dict[str, Any]):
+        cls = JSONConvertible.SUBCLASSES_BY_NAME[data['type']]
+        if cls.from_json is JSONConvertible.from_json:
+            raise NotImplementedError(f'Abstract method from_json of class {cls.__name__}')
+        return cls.from_json(data)
+
+    @classmethod
+    def schema(cls) -> Dict[str, Any]:
+        fields = cls.field_types()
+        result: Dict[str, Any] = {
+            "type": "object",
+            "properties": {'type': {'type': 'string', 'enum': [cls.__name__]}},
+            'required': ['type'],
+        }
+        for field, field_type in fields.items():
+            optional = hasattr(field_type, '__origin__') and field_type.__origin__ is typing.Union and type(None) in field_type.__args__
+            if optional:
+                field_type = Union[tuple(filter(lambda t: t is not type(None), field_type.__args__))]
+            result['properties'][field] = schema_by_python_type(field_type)
+            if not optional:
+                result['required'].append(field)
+
+        subclasses = cls.subclasses(strict=True)
+        if len(subclasses) > 0:
+            result = {'oneOf': [result] + [{'$ref': f'#/components/schemas/{s.__name__}'} for s in subclasses]}
+        return result
+
+    @classmethod
+    def schema_for_referencing(cls):
+        return {'$ref': f'#/components/schemas/{cls.__name__}'}
+
+    @classmethod
+    def schema_python_types(cls) -> Dict[str, Type]:
+        fields = typing.get_type_hints(cls.__init__)
+        result: Dict[str, Type] = {}
+        for field, field_type in fields.items():
+            result[field] = field_type
+        return result
+
+    @classmethod
+    def field_types(cls) -> Dict[str, Type]:
+        return typing.get_type_hints(cls.__init__)
+
+
+class EBC(JSONConvertible):
     def __eq__(self, other):
-        return isinstance(other, type(self)) and self.__dict__ == other.__dict__
+        return type(other) == type(self) and self.__dict__ == other.__dict__
 
     def __str__(self):
         return str(self.__dict__)
@@ -1039,18 +1028,33 @@ class EBC:
         return f'{type(self).__name__}(**' + str(self.__dict__) + ')'
 
     def to_json(self) -> Dict[str, Any]:
+        def obj_to_json(o):
+            if isinstance(o, JSONConvertible):
+                return o.to_json()
+            elif isinstance(o, numpy.ndarray):
+                return array_to_json(o)
+            elif isinstance(o, list):
+                return list_to_json(o)
+            elif isinstance(o, dict):
+                return dict_to_json(o)
+            else:
+                return o  # probably a primitive type
+
+        def dict_to_json(d: Dict[str, Any]):
+            return {k: obj_to_json(v) for k, v in d.items()}
+
+        def list_to_json(l: list):
+            return [obj_to_json(v) for v in l]
+
+        def array_to_json(o: numpy.ndarray):
+            return o.tolist()
+
         result: Dict[str, Any] = {
             'type': type(self).__name__,
             **self.__dict__,
         }
         for k in result:
-            if isinstance(result[k], EBC):
-                result[k] = result[k].to_json()
-            elif isinstance(result[k], numpy.ndarray):
-                result[k] = result[k].tolist()
-            elif isinstance(result[k], list):
-                result[k] = [r.to_json() if isinstance(r, EBC) else r
-                             for r in result[k]]
+            result[k] = obj_to_json(result[k])
         return result
 
     @staticmethod
@@ -1059,6 +1063,85 @@ class EBC:
         return class_from_json(cls, data)
 
 
+class _Container(EBC):
+    def __init__(self, item):
+        self.item = item
+
+
+def container_to_json(obj: Union[list, dict, numpy.ndarray]) -> Dict[str, Any]:
+    # Useful when converting list or dicts of json-convertible objects to json (recursively)
+    e = _Container(obj)
+    return e.to_json()['item']
+
+
+class OpenAPISchema:
+    def __init__(self, schema: dict):
+        self.schema = schema
+
+
+def schema_by_python_type(t: Union[Type, dict, OpenAPISchema]):
+    if t == 'TODO':
+        return t
+    if isinstance(t, dict):
+        return {
+            "type": "object",
+            "properties": {n: schema_by_python_type(t2) for n, t2 in t.items()},
+            'required': [n for n in t],
+        }
+    elif isinstance(t, OpenAPISchema):
+        return t.schema
+    elif t is int:
+        return {'type': 'integer'}
+    elif t is Any:
+        return {}
+    elif t is float:
+        return {'type': 'number'}
+    elif t is str:
+        return {'type': 'string'}
+    elif t is bool:
+        return {'type': 'boolean'}
+    elif t is type(None) or t is None:
+        return {'type': 'string', 'nullable': True}
+    elif hasattr(t, '__origin__'):  # probably a generic typing-type
+        if t.__origin__ is list:
+            return {'type': 'array', 'items': schema_by_python_type(t.__args__[0])}
+        elif t.__origin__ is dict:
+            return {'type': 'object', 'additionalProperties': schema_by_python_type(t.__args__[1])}
+        elif t.__origin__ is tuple and len(set(t.__args__)) == 1:
+            return {'type': 'array', 'items': schema_by_python_type(t.__args__[0]), 'minItems': len(t.__args__), 'maxItems': len(t.__args__)}
+        elif t.__origin__ is Union:
+            return {'oneOf': [schema_by_python_type(t2) for t2 in t.__args__]}
+        elif t.__origin__ is Literal:
+            return {'enum': list(t.__args__)}
+        else:
+            raise ValueError(f'Unknown type {t}')
+    elif issubclass(t, list):
+        return {'type': 'array'}
+    elif issubclass(t, type) and len(set(t.__args__)) == 1:
+        return {'type': 'array', 'items': schema_by_python_type(t.__args__[0]), 'minItems': len(t.__args__), 'maxItems': len(t.__args__)}
+    elif t is datetime.datetime:
+        from lib.date_time import SerializableDateTime
+        return schema_by_python_type(SerializableDateTime)
+    elif hasattr(t, 'schema_for_referencing'):
+        return t.schema_for_referencing()
+    else:
+        raise ValueError(f'Unknown type {t}')
+
+
+class Synchronizable:
+    def __init__(self):
+        self._lock = threading.RLock()
+
+    @staticmethod
+    def synchronized(f):
+        @functools.wraps(f)
+        def wrapper(self, *args, **kwargs):
+            with self._lock:
+                return f(self, *args, **kwargs)
+
+        return wrapper
+
+
 def class_from_json(cls, data: Dict[str, Any]):
     if isinstance(data, str):
         data = json.loads(data)
@@ -1066,30 +1149,39 @@ def class_from_json(cls, data: Dict[str, Any]):
     try:
         return cls(**data)
     except TypeError as e:
-        if "__init__() got an unexpected keyword argument 'type'" in str(e) or 'takes no arguments' in str(e):
+        if "__init__() got an unexpected keyword argument 'type'" in str(e):
             # probably this was from a to_json method
             if data['type'] != cls.__name__:
                 t = data['type']
                 logging.warning(f'Reconstructing a {cls.__name__} from a dict with type={t}')
             data = data.copy()
             del data['type']
-            for k,v  in data.items():
-                if probably_serialized_from_ebc(v):
-                    data[k] = EBC.SUBCLASSES_BY_NAME[v['type']].from_json(v)
+            for k, v in data.items():
+                if probably_serialized_from_json_convertible(v):
+                    data[k] = JSONConvertible.SUBCLASSES_BY_NAME[v['type']].from_json(v)
                 elif isinstance(v, list):
-                    data[k] = [EBC.SUBCLASSES_BY_NAME[x['type']].from_json(x)
-                         if probably_serialized_from_ebc(x)
-                         else x
-                         for x in v]
-            return allow_additional_unused_keyword_arguments(cls)(**data)
-        else:
-            raise
+                    data[k] = [JSONConvertible.SUBCLASSES_BY_NAME[x['type']].from_json(x)
+                               if probably_serialized_from_json_convertible(x)
+                               else x
+                               for x in v]
+                elif isinstance(v, dict):
+                    data[k] = {
+                        k2: JSONConvertible.SUBCLASSES_BY_NAME[x['type']].from_json(x)
+                        if probably_serialized_from_json_convertible(x)
+                        else x
+                        for k2, x in v.items()
+                    }
+            try:
+                return cls(**data)
+            except TypeError:
+                return allow_additional_unused_keyword_arguments(cls)(**data)
 
-def probably_serialized_from_ebc(data):
-    return isinstance(data, dict) and 'type' in data and data['type'] in EBC.SUBCLASSES_BY_NAME
 
+def probably_serialized_from_json_convertible(data):
+    return isinstance(data, dict) and 'type' in data and data['type'] in JSONConvertible.SUBCLASSES_BY_NAME
 
-class EBE(Enum):
+
+class EBE(JSONConvertible, Enum):
     def __int__(self):
         return self.value
 
@@ -1099,29 +1191,61 @@ class EBE(Enum):
     def __repr__(self):
         return self.name
 
+    def __lt__(self, other):
+        return list(type(self)).index(self) < list(type(self)).index(other)
+
+    def __ge__(self, other):
+        return list(type(self)).index(self) >= list(type(self)).index(other)
+
     @classmethod
-    def from_name(cls, variable_name):
+    def from_name(cls, variable_name) -> typing.Self:
         return cls.__dict__[variable_name]
 
+    def to_json(self) -> Dict[str, Any]:
+        return {
+            'type': type(self).__name__,
+            'name': self.name,
+        }
 
-class Bunch(dict, EBC):
-    def __init__(self, **kwargs):
-        dict.__init__(self, kwargs)
-        self.__dict__.update(kwargs)
+    @classmethod
+    def schema(cls) -> Dict[str, Any]:
+        return {
+            "type": "object",
+            "properties": {'type': schema_by_python_type(str), 'name': schema_by_python_type(str)}
+        }
 
-    def add_method(self, m):
-        setattr(self, m.__name__, functools.partial(m, self))
+    @staticmethod
+    def from_json(data: Dict[str, Any]) -> 'JSONConvertible':
+        cls = EBE.SUBCLASSES_BY_NAME[data['type']]
+        return cls.from_name(data['name'])
 
 
 def floor_to_multiple_of(x, multiple_of):
     return math.floor(x / multiple_of) * multiple_of
 
 
-def round_to_multiple_of(x, multiple_of):
-    return round(x / multiple_of) * multiple_of
-
-
 def ceil_to_multiple_of(x, multiple_of):
     return math.ceil(x / multiple_of) * multiple_of
 
 
+def call_tool(command, cwd=None, shell=False):
+    try:
+        print(f'Calling `{" ".join(command)}`...')
+        sub_env = os.environ.copy()
+        output: bytes = check_output(command, stderr=PIPE, env=sub_env, cwd=cwd, shell=shell)
+        output: str = output.decode('utf-8', errors='ignore')
+        return output
+    except CalledProcessError as e:
+        stdout = e.stdout.decode('utf-8', errors='ignore')
+        stderr = e.stderr.decode('utf-8', errors='ignore')
+        if len(stdout) == 0:
+            print('stdout was empty.')
+        else:
+            print('stdout was: ')
+            print(stdout)
+        if len(stderr) == 0:
+            print('stderr was empty.')
+        else:
+            print('stderr was: ')
+            print(stderr)
+        raise