From 168568bb6eb24a51fe0334b05f177524d61490c7 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Thu, 2 Oct 2025 21:33:34 +0000 Subject: [PATCH 01/11] Work on dynamix sampling. --- src/verifai/features/features.py | 55 ++++++++++++++++++------- src/verifai/samplers/feature_sampler.py | 47 +++++++++++++++------ tests/test_samplers.py | 23 ++++++++++- 3 files changed, 97 insertions(+), 28 deletions(-) diff --git a/src/verifai/features/features.py b/src/verifai/features/features.py index 8634ceb..bb89503 100644 --- a/src/verifai/features/features.py +++ b/src/verifai/features/features.py @@ -964,19 +964,26 @@ def distance(self, valueA, valueB): else: return self.distanceMetric(valueA, valueB) - @cached_property - def fixedDomains(self): + @staticmethod + def _timeExpandDomain(domain, timeBound): + return domain + + def fixedDomains(self, timeBound): """Return the fixed-length Domains associated with this feature.""" + timeExpandedDomain = self._timeExpandDomain(self.domain, timeBound) + if not self.lengthDomain: - return self.domain - domains = {} - for length in self.lengthDomain: - length = length[0] - domains[length] = Array(self.domain, (length,)) + domains = timeExpandedDomain + else: + domains = {} + for length in self.lengthDomain: + length = length[0] + domains[length] = Array(timeExpandedDomain, (length,)) + return domains def __repr__(self): - rep = f'Feature({self.domain}' + rep = f'{self.__class__.__name__}({self.domain}' if self.distribution is not None: rep += f', distribution={self.distribution}' if self.lengthDomain is not None: @@ -987,6 +994,11 @@ def __repr__(self): rep += f', distanceMetric={self.distanceMetric}' return rep + ')' +class TimeSeriesFeature(Feature): + @staticmethod + def _timeExpandDomain(domain, timeBound): + return Array(domain, (timeBound,)) + ### Feature spaces class FeatureSpace: @@ -1001,30 +1013,43 @@ class FeatureSpace: }) """ - def __init__(self, features, distanceMetric=None): + def __init__(self, features, distanceMetric=None, timeBound=None): self.namedFeatures = tuple(sorted(features.items(), key=lambda i: i[0])) self.featureNamed = OrderedDict(self.namedFeatures) self.features = tuple(self.featureNamed.values()) - self.makePoint = namedtuple('SpacePoint', self.featureNamed.keys()) + + self.staticFeatureNames = OrderedDict({name: feat for name, feat in self.featureNamed.items() + if not isinstance(feat, TimeSeriesFeature)}) + self.dynamicFeatureNames = OrderedDict({name: feat for name, feat in self.featureNamed.items() + if isinstance(feat, TimeSeriesFeature)}) + + self.makePoint = namedtuple('SpacePoint', self.featureNamed) + self.makeStaticPoint = namedtuple('SpacePoint', self.staticFeatureNames) + self.makeDynamicPoint = namedtuple('SpacePoint', self.dynamicFeatureNames) + self.distanceMetric = distanceMetric + self.timeBound = timeBound @cached_property def domains(self): - """Return the domain or domains associated with this space. + """Return the expanded domain or domains associated with this space. Returns a pair consisting of the Domain of all lengths of feature lists, plus a dict mapping each (flattened) point in that Domain to the corresponding Domain of other features. If the FeatureSpace has no feature lists, then returns (None, dom) where dom is the fixed Domain - of all features. + of all features. If any Features are TimeSeriesFeatures then they are + expanded to the a max of timeBound. """ + assert self.timeBound is not None + fixedDomains = {} lengthDomains = {} variableDomains = {} for name, feature in self.namedFeatures: if feature.lengthDomain: lengthDomains[name] = feature.lengthDomain - variableDomains[name] = feature.fixedDomains + variableDomains[name] = feature.fixedDomains(self.timeBound) else: fixedDomains[name] = feature.domain if len(lengthDomains) == 0: @@ -1065,7 +1090,7 @@ def flatten(self, point, fixedDimension=False): if feature.lengthDomain: length = len(value) flattened.append(length) - fixedDomain = feature.fixedDomains[length] + fixedDomain = feature.fixedDomains(self.timeBound)[length] fixedDomain.flattenOnto(value, flattened) if fixedDimension: # add padding to maximum length sizePerElt = domain.flattenedDimension @@ -1180,7 +1205,7 @@ def unflatten(self, coords, fixedDimension=False): domain = feature.domain if feature.lengthDomain: length = next(iterator) - fixedDomain = feature.fixedDomains[length] + fixedDomain = feature.fixedDomains(self.timeBound)[length] values.append(fixedDomain.unflattenIterator(iterator)) if fixedDimension: # consume padding sizePerElt = domain.flattenedDimension diff --git a/src/verifai/samplers/feature_sampler.py b/src/verifai/samplers/feature_sampler.py index 5890505..2224c26 100644 --- a/src/verifai/samplers/feature_sampler.py +++ b/src/verifai/samplers/feature_sampler.py @@ -8,8 +8,10 @@ import dill from dotmap import DotMap import numpy as np +from abc import ABC, abstractmethod +from contextlib import contextmanager -from verifai.features import FilteredDomain +from verifai.features import FilteredDomain, TimeSeriesFeature from verifai.samplers.domain_sampler import SplitSampler, TerminationException from verifai.samplers.rejection import RejectionSampler from verifai.samplers.halton import HaltonSampler @@ -23,7 +25,7 @@ ### Samplers defined over FeatureSpaces -class FeatureSampler: +class FeatureSampler(ABC): """Abstract class for samplers over FeatureSpaces.""" def __init__(self, space): @@ -149,6 +151,7 @@ def makeDomainSampler(domain): makeRandomSampler) return LateFeatureSampler(space, RandomSampler, makeDomainSampler) + @abstractmethod def getSample(self): """Generate a sample, along with any sampler-specific info. @@ -156,8 +159,8 @@ def getSample(self): sampler-specific info, which will be passed to the `update` method after the sample is evaluated. """ - raise NotImplementedError('tried to use abstract FeatureSampler') - + pass + def update(self, sample, info, rho): """Update the state of the sampler after evaluating a sample.""" pass @@ -211,10 +214,14 @@ class LateFeatureSampler(FeatureSampler): def __init__(self, space, makeLengthSampler, makeDomainSampler): super().__init__(space) + + if any(isinstance(f, TimeSeriesFeature) for f in self.space.features) and self.space.timeBound is None: + raise RuntimeError("Used LateFeatureSampler on FeatureSpace with includes TimeSeriesFeature but has no timeBound.") + lengthDomain, fixedDomains = space.domains if lengthDomain is None: # space has no feature lists self.lengthSampler = None - self.domainSampler = makeDomainSampler(fixedDomains) + self.domainSamplers = {None: makeDomainSampler(fixedDomains)} else: self.lengthDomain = lengthDomain self.lengthSampler = makeLengthSampler(lengthDomain) @@ -224,16 +231,32 @@ def __init__(self, space, makeLengthSampler, makeDomainSampler): } self.lastLength = None - def getSample(self): + def _sampleInternal(self): if self.lengthSampler is None: - domainPoint, info = self.domainSampler.getSample() + length, info1 = None, None else: length, info1 = self.lengthSampler.getSample() - self.lastLength = length - domainPoint, info2 = self.domainSamplers[length].getSample() - info = (info1, info2) - return self.space.makePoint(*domainPoint), info - + + self.lastLength = length + domainPoint, info2 = self.domainSamplers[length].getSample() + info = (info1, info2) + + # Make static points and iterable over dynamic points + static_features = [v for v in domainPoint._asdict().items() + if v[0] in self.space.staticFeatureNames] + dynamic_features = [v for v in domainPoint._asdict().items() + if v[0] not in self.space.staticFeatureNames] + static_point = self.space.makeStaticPoint(*static_features.values()) + + dyna + + return (static_point, info) + + @contextmanager + def getSample(self): + return self._sampleInternal() + + def update(self, sample, info, rho): if self.lengthSampler is None: self.domainSampler.update(sample, info, rho) diff --git a/tests/test_samplers.py b/tests/test_samplers.py index 4389e80..8003762 100644 --- a/tests/test_samplers.py +++ b/tests/test_samplers.py @@ -3,9 +3,30 @@ import os.path from verifai.features import (Struct, Array, Box, DiscreteBox, - Feature, FeatureSpace) + Feature, TimeSeriesFeature, FeatureSpace) from verifai.samplers import RandomSampler, FeatureSampler +def test_feature_sampling(): + space = FeatureSpace({ + 'a': Feature(DiscreteBox([0, 12])), + 'b': Feature(Box((0, 1)), lengthDomain=DiscreteBox((0, 2))), + 'c': TimeSeriesFeature(Box((2,5)), lengthDomain=DiscreteBox((0,4))) + }, + timeBound=10) + sampler = FeatureSampler.randomSamplerFor(space) + + static_sample = sampler.getSample() + + static_dict = static_sample[0]._asdict() + assert "a" in static_dict + assert "b" in static_dict + assert "c" not in static_dict + + assert len(static_dict["a"]) == 1 + assert 0 <= static_dict["a"][0] <= 12 + assert 0 <= len(static_dict["b"]) <= 2 + assert all(0 <= v <= 1 for v in static_dict["b"]) + ## Random sampling def test_domain_random(): From d1f4b466113b09c69a2cce1506d69f259c2aed18 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Fri, 3 Oct 2025 15:35:47 +0000 Subject: [PATCH 02/11] Added LateFeatureSampler for TimeSeriesFeature. --- src/verifai/features/features.py | 12 ++++----- src/verifai/samplers/feature_sampler.py | 36 +++++++++++++++++++------ tests/test_samplers.py | 25 ++++++++++++++--- 3 files changed, 55 insertions(+), 18 deletions(-) diff --git a/src/verifai/features/features.py b/src/verifai/features/features.py index bb89503..6abe1a1 100644 --- a/src/verifai/features/features.py +++ b/src/verifai/features/features.py @@ -1018,14 +1018,14 @@ def __init__(self, features, distanceMetric=None, timeBound=None): self.featureNamed = OrderedDict(self.namedFeatures) self.features = tuple(self.featureNamed.values()) - self.staticFeatureNames = OrderedDict({name: feat for name, feat in self.featureNamed.items() + self.staticFeatureNamed = OrderedDict({name: feat for name, feat in self.featureNamed.items() if not isinstance(feat, TimeSeriesFeature)}) - self.dynamicFeatureNames = OrderedDict({name: feat for name, feat in self.featureNamed.items() + self.dynamicFeatureNamed = OrderedDict({name: feat for name, feat in self.featureNamed.items() if isinstance(feat, TimeSeriesFeature)}) self.makePoint = namedtuple('SpacePoint', self.featureNamed) - self.makeStaticPoint = namedtuple('SpacePoint', self.staticFeatureNames) - self.makeDynamicPoint = namedtuple('SpacePoint', self.dynamicFeatureNames) + self.makeStaticPoint = namedtuple('SpacePoint', self.staticFeatureNamed) + self.makeDynamicPoint = namedtuple('SpacePoint', self.dynamicFeatureNamed) self.distanceMetric = distanceMetric self.timeBound = timeBound @@ -1041,7 +1041,7 @@ def domains(self): of all features. If any Features are TimeSeriesFeatures then they are expanded to the a max of timeBound. """ - assert self.timeBound is not None + assert len(self.dynamicFeatureNamed) == 0 or self.timeBound is not None fixedDomains = {} lengthDomains = {} @@ -1051,7 +1051,7 @@ def domains(self): lengthDomains[name] = feature.lengthDomain variableDomains[name] = feature.fixedDomains(self.timeBound) else: - fixedDomains[name] = feature.domain + fixedDomains[name] = feature._timeExpandDomain(feature.domain, self.timeBound) if len(lengthDomains) == 0: return (None, Struct(fixedDomains)) lengthDomain = Struct(lengthDomains) diff --git a/src/verifai/samplers/feature_sampler.py b/src/verifai/samplers/feature_sampler.py index 2224c26..aad5a8b 100644 --- a/src/verifai/samplers/feature_sampler.py +++ b/src/verifai/samplers/feature_sampler.py @@ -243,16 +243,36 @@ def _sampleInternal(self): # Make static points and iterable over dynamic points static_features = [v for v in domainPoint._asdict().items() - if v[0] in self.space.staticFeatureNames] + if v[0] in self.space.staticFeatureNamed] dynamic_features = [v for v in domainPoint._asdict().items() - if v[0] not in self.space.staticFeatureNames] - static_point = self.space.makeStaticPoint(*static_features.values()) + if v[0] not in self.space.staticFeatureNamed] + static_point = self.space.makeStaticPoint(*[v[1] for v in static_features]) + + dynamic_points = [] + for t in range(self.space.timeBound): + point_dict = {} + + for f, val in dynamic_features: + if not self.space.featureNamed[f].lengthDomain: + point_dict[f] = val[t] + else: + feat_list = [] + for l in range(len(val)): + feat_list.append(val[l][t]) + point_dict[f] = tuple(feat_list) + + dynamic_points.append(self.space.makeDynamicPoint(*point_dict.values())) + + def gen_dynamic_points(): + try: + for p in dynamic_points: + yield p + raise RuntimeError("Exceeded provided timeBound") + finally: + pass + + return (static_point, info), gen_dynamic_points() - dyna - - return (static_point, info) - - @contextmanager def getSample(self): return self._sampleInternal() diff --git a/tests/test_samplers.py b/tests/test_samplers.py index 8003762..82e4a4c 100644 --- a/tests/test_samplers.py +++ b/tests/test_samplers.py @@ -10,22 +10,39 @@ def test_feature_sampling(): space = FeatureSpace({ 'a': Feature(DiscreteBox([0, 12])), 'b': Feature(Box((0, 1)), lengthDomain=DiscreteBox((0, 2))), - 'c': TimeSeriesFeature(Box((2,5)), lengthDomain=DiscreteBox((0,4))) + 'c': TimeSeriesFeature(Box((2,5))), + 'd': TimeSeriesFeature(Box((5,6)), lengthDomain=DiscreteBox((0,2))) }, timeBound=10) sampler = FeatureSampler.randomSamplerFor(space) - static_sample = sampler.getSample() + static_sample, dynamic_points_gen = sampler.getSample() + static_point = static_sample[0] - static_dict = static_sample[0]._asdict() + static_dict = static_point._asdict() assert "a" in static_dict assert "b" in static_dict assert "c" not in static_dict + assert "d" not in static_dict assert len(static_dict["a"]) == 1 assert 0 <= static_dict["a"][0] <= 12 assert 0 <= len(static_dict["b"]) <= 2 - assert all(0 <= v <= 1 for v in static_dict["b"]) + assert all(0 <= v[0] <= 1 for v in static_dict["b"]) + + for _ in range(space.timeBound): + dynamic_point = next(dynamic_points_gen) + print(dynamic_point) + dynamic_dict = dynamic_point._asdict() + assert "a" not in dynamic_dict + assert "b" not in dynamic_dict + assert "c" in dynamic_dict + assert "d" in dynamic_dict + + assert len(dynamic_dict["c"]) == 1 + assert 2 <= dynamic_dict["c"][0] <= 5 + assert 0 <= len(dynamic_dict["d"]) <= 2 + assert all(5 <= v[0] <= 6 for v in dynamic_dict["d"]) ## Random sampling From f12ca280ef872bd508b60fb3e1e8b979e7becf68 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Wed, 12 Nov 2025 17:42:17 -0800 Subject: [PATCH 03/11] Mostly working dynamic sampling. --- src/verifai/features/features.py | 169 +++++++++++++++++--- src/verifai/samplers/domain_sampler.py | 26 +-- src/verifai/samplers/feature_sampler.py | 157 ++++++++++-------- src/verifai/samplers/scenic_sampler.py | 48 +++++- src/verifai/server.py | 7 +- tests/scenic/scenic_driving_behavior.scenic | 16 ++ tests/scenic/test_scenic.py | 42 +++-- tests/test_crossEntropy.py | 8 +- tests/test_examples.py | 4 +- tests/test_features.py | 44 ++--- tests/test_grid.py | 10 +- tests/test_halton.py | 2 +- tests/test_samplers.py | 20 ++- tests/test_simulatedAnnealing.py | 2 +- tests/utils.py | 37 +++-- 15 files changed, 418 insertions(+), 174 deletions(-) create mode 100644 tests/scenic/scenic_driving_behavior.scenic diff --git a/src/verifai/features/features.py b/src/verifai/features/features.py index 6abe1a1..49ab6d5 100644 --- a/src/verifai/features/features.py +++ b/src/verifai/features/features.py @@ -1013,7 +1013,7 @@ class FeatureSpace: }) """ - def __init__(self, features, distanceMetric=None, timeBound=None): + def __init__(self, features, distanceMetric=None, timeBound=0): self.namedFeatures = tuple(sorted(features.items(), key=lambda i: i[0])) self.featureNamed = OrderedDict(self.namedFeatures) self.features = tuple(self.featureNamed.values()) @@ -1023,13 +1023,16 @@ def __init__(self, features, distanceMetric=None, timeBound=None): self.dynamicFeatureNamed = OrderedDict({name: feat for name, feat in self.featureNamed.items() if isinstance(feat, TimeSeriesFeature)}) - self.makePoint = namedtuple('SpacePoint', self.featureNamed) - self.makeStaticPoint = namedtuple('SpacePoint', self.staticFeatureNamed) - self.makeDynamicPoint = namedtuple('SpacePoint', self.dynamicFeatureNamed) + # self.makePoint = namedtuple('SpacePoint', self.featureNamed) + self.makeStaticPoint = namedtuple('StaticSpacePoint', self.staticFeatureNamed) + self.makeDynamicPoint = namedtuple('DynamicSpacePoint', self.dynamicFeatureNamed) self.distanceMetric = distanceMetric self.timeBound = timeBound + if len(self.dynamicFeatureNamed) > 0 and self.timeBound == 0: + raise RuntimeError("FeatureSpace which includes TimeSeriesFeature has no timeBound.") + @cached_property def domains(self): """Return the expanded domain or domains associated with this space. @@ -1041,8 +1044,6 @@ def domains(self): of all features. If any Features are TimeSeriesFeatures then they are expanded to the a max of timeBound. """ - assert len(self.dynamicFeatureNamed) == 0 or self.timeBound is not None - fixedDomains = {} lengthDomains = {} variableDomains = {} @@ -1081,11 +1082,14 @@ def flatten(self, point, fixedDimension=False): """Flatten a point in this space. See Domain.flatten. If fixedDimension is True, the point is flattened out as if all feature - lists had their maximum lengths, with None as a placeholder. This means - that all points in the space will flatten to the same length. + lists had their maximum lengths and time steps, with None as a placeholder. + This means that all points in the space will flatten to the same length. """ + from verifai.samplers.feature_sampler import Sample + assert isinstance(point, Sample) + flattened = [] - for feature, value in zip(self.features, point): + for feature, value in zip(self.staticFeatureNamed.values(), point.staticSample): domain = feature.domain if feature.lengthDomain: length = len(value) @@ -1099,6 +1103,39 @@ def flatten(self, point, fixedDimension=False): flattened.append(None) else: domain.flattenOnto(value, flattened) + + flattened.append(len(point.dynamicSampleHistory)) + + for feature_i, feature in enumerate(self.dynamicFeatureNamed.values()): + if feature.lengthDomain: + length = len(value) + else: + length = None + + flattened.append(length) + + for dynamic_point in point.dynamicSampleHistory: + value = dynamic_point[feature_i] + domain = feature.domain + + if length is None: + domain.flattenOnto(value, flattened) + else: + fixedDomain = feature.fixedDomains(self.timeBound)[length] + fixedDomain.flattenOnto(value, flattened) + if fixedDimension: + sizePerElt = domain.flattenedDimension + needed = (feature.maxLength - length) * sizePerElt + for i in range(needed): + flattened.append(None) + + if fixedDimension: + needed = (self.space.timeBound - len(point.dynamicSampleHistory)) * length * sizePerElt + flattened += [None for _ in range(needed)] + + flattened_point = tuple(flattened) + if fixedDimension: + assert len(flattened_point) == self.fixedFlattenedDimension return tuple(flattened) @cached_property @@ -1108,13 +1145,15 @@ def fixedFlattenedDimension(self): Also an upper bound on the length of the vector returned by flatten by default, when fixedDimension=False.""" dim = 0 + dim += 1 # Timesteps for feature in self.features: domain = feature.domain + timeMult = self.timeBound if isinstance(feature, TimeSeriesFeature) else 1 if feature.lengthDomain: dim += 1 # dimension storing length of the feature list - dim += feature.maxLength * domain.flattenedDimension + dim += timeMult * feature.maxLength * domain.flattenedDimension else: - dim += domain.flattenedDimension + dim += timeMult * domain.flattenedDimension return dim def meaningOfFlatCoordinate(self, index, pointName='point'): @@ -1125,26 +1164,53 @@ def meaningOfFlatCoordinate(self, index, pointName='point'): have different meaning depending on the lengths of feature lists. """ assert 0 <= index < self.fixedFlattenedDimension - for name, feature in self.namedFeatures: + for name, feature in self.staticFeatureNamed.items(): domain = feature.domain if feature.lengthDomain: if index == 0: - return f'len({pointName}.{name})' + return f'len({pointName}.staticSample.{name})' else: index -= 1 elem = index // domain.flattenedDimension if elem < feature.maxLength: subIndex = index % domain.flattenedDimension - subPoint = f'{pointName}.{name}[{elem}]' + subPoint = f'{pointName}.staticSample.{name}[{elem}]' return domain.meaningOfFlatCoordinate(subIndex, pointName=subPoint) index -= feature.maxLength * domain.flattenedDimension else: if index < domain.flattenedDimension: - subPoint = f'{pointName}.{name}' + subPoint = f'{pointName}.staticSample.{name}' return domain.meaningOfFlatCoordinate(index, pointName=subPoint) index -= domain.flattenedDimension + + if index == 0: + return f'len({pointName}.dynamicSampleHistory)' + index -= 1 + + for name, feature in self.dynamicFeatureNamed.items(): + domain = feature.domain + for time_i in range(self.timeBound): + if feature.lengthDomain: + if index == 0: + return f'len({pointName}.dynamicSampleHistory[{time_i}].{name})' + else: + index -= 1 + elem = index // domain.flattenedDimension + if elem < feature.maxLength: + subIndex = index % domain.flattenedDimension + subPoint = f'{pointName}.dynamicSampleHistory[{time_i}].{name}[{elem}]' + return domain.meaningOfFlatCoordinate(subIndex, + pointName=subPoint) + index -= feature.maxLength * domain.flattenedDimension + else: + if index < domain.flattenedDimension: + subPoint = f'{pointName}.dynamicSampleHistory[{time_i}].{name}' + return domain.meaningOfFlatCoordinate(index, + pointName=subPoint) + index -= domain.flattenedDimension + raise RuntimeError('impossible index arithmetic') def pandasIndexForFlatCoordinate(self, index): @@ -1152,6 +1218,7 @@ def pandasIndexForFlatCoordinate(self, index): See meaningOfFlatCoordinate, and Domain.pandasIndexForFlatCoordinate. """ + #TODO Update assert 0 <= index < self.fixedFlattenedDimension for name, feature in self.namedFeatures: domain = feature.domain @@ -1179,7 +1246,7 @@ def coordinateIsNumerical(self, index): See meaningOfFlatCoordinate, and Domain.coordinateIsNumerical. """ assert 0 <= index < self.fixedFlattenedDimension - for name, feature in self.namedFeatures: + for name, feature in self.staticFeatureNamed.items(): domain = feature.domain if feature.lengthDomain: if index == 0: @@ -1195,26 +1262,86 @@ def coordinateIsNumerical(self, index): if index < domain.flattenedDimension: return domain.coordinateIsNumerical(index) index -= domain.flattenedDimension + + if index == 0: + return True + index -= 1 + + for name, feature in self.dynamicFeatureNamed.items(): + domain = feature.domain + for time_i in range(self.timeBound): + if feature.lengthDomain: + if index == 0: + return True + else: + index -= 1 + elem = index // domain.flattenedDimension + if elem < feature.maxLength: + subIndex = index % domain.flattenedDimension + return domain.coordinateIsNumerical(subIndex) + index -= feature.maxLength * domain.flattenedDimension + else: + if index < domain.flattenedDimension: + return domain.coordinateIsNumerical(index) + index -= domain.flattenedDimension + raise RuntimeError('impossible index arithmetic') def unflatten(self, coords, fixedDimension=False): """Unflatten a tuple of coordinates to a point in this space.""" - values = [] + from verifai.samplers.feature_sampler import LateFeatureSample + + staticValues = [] iterator = iter(coords) - for feature in self.features: + + for feature in self.staticFeatureNamed.values(): domain = feature.domain if feature.lengthDomain: length = next(iterator) fixedDomain = feature.fixedDomains(self.timeBound)[length] - values.append(fixedDomain.unflattenIterator(iterator)) + staticValues.append(fixedDomain.unflattenIterator(iterator)) if fixedDimension: # consume padding sizePerElt = domain.flattenedDimension needed = (feature.maxLength - length) * sizePerElt for i in range(needed): next(iterator) else: - values.append(domain.unflattenIterator(iterator)) - return self.makePoint(*values) + staticValues.append(domain.unflattenIterator(iterator)) + + staticSample = self.makeStaticPoint(*staticValues) + + timeSteps = next(iterator) + + dynamicValuesList = [[] for _ in range(timeSteps)] + + for feature in self.dynamicFeatureNamed.values(): + domain = feature.domain + length = next(iterator) + + for time_i in range(timeSteps): + + + if length is None: + dynamicValuesList[time_i].append(domain.unflattenIterator(iterator)) + else: + fixedDomain = feature.fixedDomains(self.timeBound)[length] + dynamicValuesList[time_i].append(fixedDomain.unflattenIterator(iterator)) + if fixedDimension: # consume padding + sizePerElt = domain.flattenedDimension + needed = (feature.maxLength - length) * sizePerElt + for i in range(needed): + next(iterator) + + if fixedDimension: + needed = (self.space.timeBound - timeSteps) * length * sizePerElt + for i in range(needed): + next(iterator) + + dynamicSampleList = [self.makeDynamicPoint(*dynamicValues) for dynamicValues in dynamicValuesList] + + updateCallback = lambda rho: None + + return LateFeatureSample(space=None, staticSample=staticSample, dynamicSampleList=dynamicSampleList, updateCallback=updateCallback) def __repr__(self): rep = f'FeatureSpace({self.featureNamed}' diff --git a/src/verifai/samplers/domain_sampler.py b/src/verifai/samplers/domain_sampler.py index 6ead563..83426ae 100644 --- a/src/verifai/samplers/domain_sampler.py +++ b/src/verifai/samplers/domain_sampler.py @@ -47,22 +47,24 @@ def update(self, sample, info, rho): """ pass - def nextSample(self, feedback=None): - """Generate the next sample, given feedback from the last sample. - - This exists only for backwards-compatibility. It has been replaced by - the getSample and update APIs. - """ - if self.last_sample is not None: - self.update(self.last_sample, self.last_info, feedback) - self.last_sample, self.last_info = self.getSample() - return self.last_sample + # TODO: Deprecate + # def nextSample(self, feedback=None): + # """Generate the next sample, given feedback from the last sample. + + # This exists only for backwards-compatibility. It has been replaced by + # the getSample and update APIs. + # """ + # if self.last_sample is not None: + # self.update(self.last_sample, self.last_info, feedback) + # self.last_sample, self.last_info = self.getSample() + # return self.last_sample def __iter__(self): try: - feedback = None while True: - feedback = yield self.nextSample(feedback) + sample, info = self.getSample() + rho = yield sample + self.update(sample, info, rho) except TerminationException: return diff --git a/src/verifai/samplers/feature_sampler.py b/src/verifai/samplers/feature_sampler.py index aad5a8b..e79fd69 100644 --- a/src/verifai/samplers/feature_sampler.py +++ b/src/verifai/samplers/feature_sampler.py @@ -23,6 +23,29 @@ from verifai.samplers.simulated_annealing import SimulatedAnnealingSampler from verifai.samplers.grid_sampler import GridSampler +class Sample(ABC): + def __init__(self, space): + self.space = space + self.dynamicSampleHistory = [] + + @property + @abstractmethod + def staticSample(self): + pass + + @abstractmethod + def _getDynamicSample(self, info): + pass + + def getDynamicSample(self, info=None): + sample = self._getDynamicSample(info) + self.dynamicSampleHistory.append(sample) + return sample + + @abstractmethod + def update(self, rho): + pass + ### Samplers defined over FeatureSpaces class FeatureSampler(ABC): @@ -151,30 +174,14 @@ def makeDomainSampler(domain): makeRandomSampler) return LateFeatureSampler(space, RandomSampler, makeDomainSampler) - @abstractmethod - def getSample(self): - """Generate a sample, along with any sampler-specific info. - - Must return a pair consisting of the sample and arbitrary - sampler-specific info, which will be passed to the `update` - method after the sample is evaluated. - """ - pass - - def update(self, sample, info, rho): + def update(self, sample_id, rho): """Update the state of the sampler after evaluating a sample.""" pass - def nextSample(self, feedback=None): - """Generate the next sample, given feedback from the last sample. - - This function exists only for backwards compatibility. It has been - superceded by the `getSample` and `update` APIs. - """ - if self.last_sample is not None: - self.update(self.last_sample, self.last_info, feedback) - self.last_sample, self.last_info = self.getSample() - return self.last_sample + @abstractmethod + def getSample(self): + """Returns a `Sample` object""" + pass def set_graph(self, graph): self.scenario.set_graph(graph) @@ -197,12 +204,42 @@ def restoreFromFile(path): def __iter__(self): try: - feedback = None while True: - feedback = yield self.nextSample(feedback) + sample = self.getSample() + rho = yield sample + sample.update(rho) except TerminationException: return +class LateFeatureSample(Sample): + def __init__(self, space, staticSample, dynamicSampleList, updateCallback): + super().__init__(space) + self._staticSample = staticSample + self._dynamicSampleList = dynamicSampleList + self._updateCallback = updateCallback + self._i = 0 + + @property + def staticSample(self): + return self._staticSample + + def _getDynamicSample(self, info): + if self.space.timeBound == 0: + raise RuntimeError("Called `getDynamicSample` with `timeBound` of `FeatureSpace` set to 0") + + if self._i >= self.space.timeBound: + raise RuntimeError("Exceeded `timeBound` of `FeatureSpace`") + + assert self._i < len(self._dynamicSampleList) + + dynamic_sample = self._dynamicSampleList[self._i] + self._i += 1 + + return dynamic_sample + + def update(self, rho): + return self._updateCallback(rho) + class LateFeatureSampler(FeatureSampler): """FeatureSampler that works by first sampling only lengths of feature lists, then sampling from the resulting fixed-dimensional Domain. @@ -215,9 +252,6 @@ class LateFeatureSampler(FeatureSampler): def __init__(self, space, makeLengthSampler, makeDomainSampler): super().__init__(space) - if any(isinstance(f, TimeSeriesFeature) for f in self.space.features) and self.space.timeBound is None: - raise RuntimeError("Used LateFeatureSampler on FeatureSpace with includes TimeSeriesFeature but has no timeBound.") - lengthDomain, fixedDomains = space.domains if lengthDomain is None: # space has no feature lists self.lengthSampler = None @@ -229,17 +263,26 @@ def __init__(self, space, makeLengthSampler, makeDomainSampler): point: makeDomainSampler(domain) for point, domain in fixedDomains.items() } - self.lastLength = None - def _sampleInternal(self): + self.id_metadata_dict = {} + self._last_id = 0 + + def get_info_id(self, info, length, sample): + self._last_id += 1 + self.id_metadata_dict[self._last_id] = (info, length, sample) + return self._last_id + + def getSample(self): if self.lengthSampler is None: length, info1 = None, None else: length, info1 = self.lengthSampler.getSample() - self.lastLength = length domainPoint, info2 = self.domainSamplers[length].getSample() info = (info1, info2) + + sample_id = self.get_info_id(info, length, domainPoint) + update_callback = lambda rho: self.update(sample_id, rho) # Make static points and iterable over dynamic points static_features = [v for v in domainPoint._asdict().items() @@ -249,46 +292,32 @@ def _sampleInternal(self): static_point = self.space.makeStaticPoint(*[v[1] for v in static_features]) dynamic_points = [] - for t in range(self.space.timeBound): - point_dict = {} - - for f, val in dynamic_features: - if not self.space.featureNamed[f].lengthDomain: - point_dict[f] = val[t] - else: - feat_list = [] - for l in range(len(val)): - feat_list.append(val[l][t]) - point_dict[f] = tuple(feat_list) - - dynamic_points.append(self.space.makeDynamicPoint(*point_dict.values())) - - def gen_dynamic_points(): - try: - for p in dynamic_points: - yield p - raise RuntimeError("Exceeded provided timeBound") - finally: - pass - - return (static_point, info), gen_dynamic_points() + if any(isinstance(f, TimeSeriesFeature) for f in self.space.features): + for t in range(self.space.timeBound): + point_dict = {} - def getSample(self): - return self._sampleInternal() - + for f, val in dynamic_features: + if not self.space.featureNamed[f].lengthDomain: + point_dict[f] = val[t] + else: + feat_list = [] + for l in range(len(val)): + feat_list.append(val[l][t]) + point_dict[f] = tuple(feat_list) + + dynamic_points.append(self.space.makeDynamicPoint(*point_dict.values())) + + return LateFeatureSample(self.space, static_point, dynamic_points, update_callback) + + def update(self, sample_id, rho): + info, lengthPoint, domainPoint = self.id_metadata_dict[sample_id] - def update(self, sample, info, rho): if self.lengthSampler is None: - self.domainSampler.update(sample, info, rho) + self.domainSamplers[None].update(domainPoint, info[1], rho) else: - self.lengthSampler.update(sample, info[0], rho) - lengths = [] - for name, feature in self.space.namedFeatures: - if feature.lengthDomain: - lengths.append((len(getattr(sample, name)),)) - lengthPoint = self.lengthDomain.makePoint(*lengths) - self.domainSamplers[lengthPoint].update(sample, info[1], rho) + self.lengthSampler.update(domainPoint, info[0], rho) + self.domainSamplers[lengthPoint].update(domainPoint, info[1], rho) ### Utilities def makeRandomSampler(domain): diff --git a/src/verifai/samplers/scenic_sampler.py b/src/verifai/samplers/scenic_sampler.py index bcea7fa..45e4a11 100644 --- a/src/verifai/samplers/scenic_sampler.py +++ b/src/verifai/samplers/scenic_sampler.py @@ -16,7 +16,7 @@ from verifai.features import (Constant, Categorical, Real, Box, Array, Struct, Feature, FeatureSpace) -from verifai.samplers.feature_sampler import FeatureSampler +from verifai.samplers.feature_sampler import FeatureSampler, Sample from verifai.utils.frozendict import frozendict scenicMajorVersion = int(importlib.metadata.version('scenic').split('.')[0]) @@ -223,6 +223,22 @@ def spaceForScenario(scenario, ignoredProperties): }) return space, quotedParams +class ScenicSample(Sample): + def __init__(self, space, staticSample, updateCallback): + super().__init__(space) + self._staticSample = staticSample + self._updateCallback = updateCallback + + @property + def staticSample(self): + return self._staticSample + + def _getDynamicSample(self, info): + raise RuntimeError("ScenicSample does not support dynamic sampling.") + + def update(self, rho): + self._updateCallback(rho) + class ScenicSampler(FeatureSampler): """Samples from the induced distribution of a Scenic scenario. @@ -236,7 +252,9 @@ class ScenicSampler(FeatureSampler): def __init__(self, scenario, maxIterations=None, ignoredProperties=None): self.scenario = scenario self.maxIterations = 2000 if maxIterations is None else maxIterations + self._nextScene = None self.lastScene = None + self.lastFeedback = None if ignoredProperties is None: ignoredProperties = defaultIgnoredProperties space, self.quotedParams = spaceForScenario(scenario, ignoredProperties) @@ -274,12 +292,32 @@ def fromScenicCode(cls, code, maxIterations=None, return cls(scenario, maxIterations=maxIterations, ignoredProperties=ignoredProperties) - def nextSample(self, feedback=None): + def getSample(self): ret = self.scenario.generate( - maxIterations=self.maxIterations, feedback=feedback, verbosity=0 + maxIterations=self.maxIterations, feedback=self.lastFeedback, verbosity=0 ) + + self.lastFeedback = None self.lastScene, _ = ret - return self.pointForScene(self.lastScene) + + staticSample = self.pointForScene(self.lastScene) + updateCallback = lambda rho: self.update(0, rho) + + return ScenicSample(self.space, staticSample, updateCallback) + + def update(self, sample_id, rho): + assert sample_id == 0 + if self.lastFeedback is not None: + raise RuntimeError("Called `update` twice in a row (ScenicSampler does not support non-sequential sampling)") + self.lastFeedback = rho + + # TODO: Deprecate + # def nextSample(self, feedback=None): + # ret = self.scenario.generate( + # maxIterations=self.maxIterations, feedback=feedback, verbosity=0 + # ) + # self.lastScene, _ = ret + # return self.pointForScene(self.lastScene) def pointForScene(self, scene): """Convert a sampled Scenic :obj:`~scenic.core.scenarios.Scene` to a point in our feature space. @@ -314,7 +352,7 @@ def pointForScene(self, scene): params[param] = pointForValue(subdom, scene.params[originalName]) paramPoint = paramDomain.makePoint(**params) - return self.space.makePoint(objects=objPoint, params=paramPoint) + return self.space.makeStaticPoint(objects=objPoint, params=paramPoint) @staticmethod def nameForObject(i): diff --git a/src/verifai/server.py b/src/verifai/server.py index ef4b043..32e5f64 100644 --- a/src/verifai/server.py +++ b/src/verifai/server.py @@ -176,8 +176,8 @@ def terminate(self): def close_connection(self): self.client_socket.close() - def get_sample(self, feedback): - return self.sampler.nextSample(feedback) + def get_sample(self): + return self.sampler.getSample() def flatten_sample(self, sample): return self.sampler.space.flatten(sample) @@ -193,9 +193,10 @@ def evaluate_sample(self, sample): def run_server(self): start = time.time() - sample = self.get_sample(self.lastValue) + sample = self.get_sample() after_sampling = time.time() self.lastValue = self.evaluate_sample(sample) + sample.update(self.lastValue) after_simulation = time.time() timings = ServerTimings(sample_time=(after_sampling - start), simulate_time=(after_simulation - after_sampling)) diff --git a/tests/scenic/scenic_driving_behavior.scenic b/tests/scenic/scenic_driving_behavior.scenic new file mode 100644 index 0000000..ab19e90 --- /dev/null +++ b/tests/scenic/scenic_driving_behavior.scenic @@ -0,0 +1,16 @@ +param map = localPath('Town01.xodr') +param carla_map = 'Town01' +param verifaiTimeBound = 100 + +model scenic.domains.driving.model + +foo = VerifaiRange(0,0.01, timeSeries=True) + +behavior TestBehavior(): + while True: + take SetThrottleAction(foo.getSample()) + +ego = new Car on road, with behavior TestBehavior() +new Car behind ego by VerifaiRange(1,4) + +terminate after 5 seconds diff --git a/tests/scenic/test_scenic.py b/tests/scenic/test_scenic.py index 2b3cf78..30efc8e 100644 --- a/tests/scenic/test_scenic.py +++ b/tests/scenic/test_scenic.py @@ -15,8 +15,8 @@ def test_objects(new_Object): f'ego = {new_Object} at 4 @ 9', maxIterations=1 ) - sample = sampler.nextSample() - objects = sample.objects + sample = sampler.getSample() + objects = sample.staticSample.objects assert len(objects) == 1 pos = objects.object0.position assert type(pos) is tuple @@ -28,7 +28,7 @@ def test_params(new_Object): f'ego = {new_Object}', maxIterations=1 ) - sample = sampler.nextSample() + sample = sampler.getSample().staticSample x = sample.params.x assert type(x) is float assert 3 <= x <= 5 @@ -39,7 +39,7 @@ def test_quoted_param(new_Object): f'ego = {new_Object}', maxIterations=1 ) - sample = sampler.nextSample() + sample = sampler.getSample().staticSample v = sampler.paramDictForSample(sample)['x/y'] assert type(v) is float assert 3 <= v <= 5 @@ -49,7 +49,7 @@ def test_lists(new_Object): f'ego = {new_Object} with foo [1, -1, 3.3]', maxIterations=1 ) - sample = sampler.nextSample() + sample = sampler.getSample().staticSample foo = sample.objects.object0.foo assert type(foo) is tuple assert foo == pytest.approx((1, -1, 3.3)) @@ -68,8 +68,8 @@ def test_object_order(new_Object): f' {new_Object} at 2*i @ 0', maxIterations=1 ) - sample = sampler.nextSample() - objects = sample.objects + sample = sampler.getSample() + objects = sample.staticSample.objects assert len(objects) == 11 for i in range(len(objects)): name = ScenicSampler.nameForObject(i) @@ -78,7 +78,7 @@ def test_object_order(new_Object): flat = sampler.space.flatten(sample) unflat = sampler.space.unflatten(flat) - assert unflat == sample + assert unflat.staticSample == sample.staticSample ## Active sampling @@ -93,9 +93,9 @@ def test_active_sampling(new_Object): maxIterations=1 ) def f(sample): - return -1 if sample.objects.object0.position[0] < 0 else 1 + return -1 if sample.staticSample.objects.object0.position[0] < 0 else 1 samples = sampleWithFeedback(sampler, 120, f) - xs = [sample.objects.object0.position[0] for sample in samples] + xs = [sample.staticSample.objects.object0.position[0] for sample in samples] assert all(-1 <= x <= 1 for x in xs) assert any(x > 0 for x in xs) assert 66 <= sum(x < 0 for x in xs[50:]) @@ -112,7 +112,7 @@ def test_active_save_restore(new_Object, tmpdir): def runSampler(sampler): for i in range(3): - sample = sampler.nextSample() + sample = sampler.getSample().staticSample print(f'Sample #{i}:') print(sample) @@ -179,3 +179,23 @@ def test_driving_dynamic(pathToLocalFile): server_class=ScenicServer, server_options=server_options) falsifier.run_falsifier() + +def test_driving_dynamic_behavior(pathToLocalFile): + path = pathToLocalFile('scenic_driving_behavior.scenic') + sampler = ScenicSampler.fromScenario( + path, + model='scenic.simulators.newtonian.driving_model', + params=dict(render=False), + mode2D=True, + ) + falsifier_params = DotMap( + n_iters=3, + save_error_table=False, + save_safe_table=False, + ) + server_options = DotMap(maxSteps=2, verbosity=3) + falsifier = generic_falsifier(sampler=sampler, + falsifier_params=falsifier_params, + server_class=ScenicServer, + server_options=server_options) + falsifier.run_falsifier() diff --git a/tests/test_crossEntropy.py b/tests/test_crossEntropy.py index 6238b2e..926e8ff 100644 --- a/tests/test_crossEntropy.py +++ b/tests/test_crossEntropy.py @@ -17,8 +17,8 @@ def test_crossEntropy(): }) def f(sample): - print(sample.cars[0].heading[0] - 0.75) - return abs(sample.cars[0].heading[0] - 0.75) + print(sample.staticSample.cars[0].heading[0] - 0.75) + return abs(sample.staticSample.cars[0].heading[0] - 0.75) ce_params = DotMap() ce_params.alpha =0.9 @@ -37,8 +37,8 @@ def test_feedback_multiple_lengths(): }) def f(sample): - assert 1 <= len(sample.a) <= 2 - return -1 if len(sample.a) == 1 and sample.a[0][0] < 0.5 else 1 + assert 1 <= len(sample.staticSample.a) <= 2 + return -1 if len(sample.staticSample.a) == 1 and sample.staticSample.a[0][0] < 0.5 else 1 ce_params = DotMap(alpha=0.5, thres=0) ce_params.cont.buckets = 2 diff --git a/tests/test_examples.py b/tests/test_examples.py index 764ae04..a03bdcd 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -15,9 +15,9 @@ def test_example(): sampler = FeatureSampler.samplerFor(space) for i in range(3): - sample = sampler.nextSample() + sample = sampler.getSample() print(f'Sample #{i}:') print(sample) flat = space.flatten(sample) unflat = space.unflatten(flat) - assert sample == unflat + assert sample.staticSample == unflat.staticSample diff --git a/tests/test_features.py b/tests/test_features.py index 3722ecc..f47b3ed 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -10,39 +10,39 @@ def test_fs_flatten(): }) sampler = FeatureSampler.randomSamplerFor(space) for i in range(100): - point = sampler.nextSample() + point = sampler.getSample() flat = space.flatten(point) assert type(flat) is tuple assert len(flat) <= space.fixedFlattenedDimension unflat = space.unflatten(flat) - assert point == unflat + assert point.staticSample == unflat.staticSample def test_fs_flatten_fixed_dimension(): space = FeatureSpace({ 'a': Feature(DiscreteBox([0, 12])), 'b': Feature(Box((0, 1)), lengthDomain=DiscreteBox((0, 2))) }) - assert space.fixedFlattenedDimension == 4 + assert space.fixedFlattenedDimension == 5 sampler = FeatureSampler.randomSamplerFor(space) for i in range(100): - point = sampler.nextSample() + point = sampler.getSample() flat = space.flatten(point, fixedDimension=True) assert type(flat) is tuple - assert len(flat) == 4 - bLen = len(point.b) - assert eval(space.meaningOfFlatCoordinate(0)) == point.a[0] + assert len(flat) == 5 + bLen = len(point.staticSample.b) + assert eval(space.meaningOfFlatCoordinate(0, pointName='point')) == point.staticSample.a[0] assert flat[1] == bLen - assert eval(space.meaningOfFlatCoordinate(1)) == bLen + assert eval(space.meaningOfFlatCoordinate(1, pointName='point')) == bLen if bLen < 1: assert flat[2] is None else: - assert eval(space.meaningOfFlatCoordinate(2)) == point.b[0][0] + assert eval(space.meaningOfFlatCoordinate(2, pointName='point')) == point.staticSample.b[0][0] if bLen < 2: assert flat[3] is None else: - assert eval(space.meaningOfFlatCoordinate(3)) == point.b[1][0] + assert eval(space.meaningOfFlatCoordinate(3, pointName='point')) == point.staticSample.b[1][0] unflat = space.unflatten(flat, fixedDimension=True) - assert point == unflat + assert point.staticSample == unflat.staticSample assert space.pandasIndexForFlatCoordinate(0) == ('a', 0) assert space.pandasIndexForFlatCoordinate(1) == ('b', 'length') assert space.pandasIndexForFlatCoordinate(2) == ('b', 0, 0) @@ -55,28 +55,28 @@ def test_fs_flatten_fixed_dimension2(): 'b': Feature(cat, lengthDomain=DiscreteBox((1, 3))), 'c': Feature(Box([-1, 1], [-3, 3])) }) - assert space.fixedFlattenedDimension == 6 + assert space.fixedFlattenedDimension == 7 sampler = FeatureSampler.randomSamplerFor(space) for i in range(100): - point = sampler.nextSample() + point = sampler.getSample() flat = space.flatten(point, fixedDimension=True) assert type(flat) is tuple - assert len(flat) == 6 - bLen = len(point.b) + assert len(flat) == 7 + bLen = len(point.staticSample.b) assert 1 <= bLen <= 3 assert flat[0] == bLen - assert eval(space.meaningOfFlatCoordinate(0)) == bLen - assert eval(space.meaningOfFlatCoordinate(1)) == point.b[0] + assert eval(space.meaningOfFlatCoordinate(0, pointName='point')) == bLen + assert eval(space.meaningOfFlatCoordinate(1, pointName='point')) == point.staticSample.b[0] if bLen < 2: assert flat[2] is None else: - assert eval(space.meaningOfFlatCoordinate(2)) == point.b[1] + assert eval(space.meaningOfFlatCoordinate(2, pointName='point')) == point.staticSample.b[1] if bLen < 3: assert flat[3] is None else: - assert eval(space.meaningOfFlatCoordinate(3)) == point.b[2] + assert eval(space.meaningOfFlatCoordinate(3, pointName='point')) == point.staticSample.b[2] unflat = space.unflatten(flat, fixedDimension=True) - assert point == unflat + assert point.staticSample == unflat.staticSample assert space.pandasIndexForFlatCoordinate(0) == ('b', 'length') assert space.pandasIndexForFlatCoordinate(1) == ('b', 0) assert space.pandasIndexForFlatCoordinate(2) == ('b', 1) @@ -91,8 +91,8 @@ def test_fs_distance(): box = Box([0, 10]) space = FeatureSpace({ 'a': Feature(box), 'b': Feature(box) }) sampler = FeatureSampler.randomSamplerFor(space) - pointA = sampler.nextSample() - pointB = sampler.nextSample() + pointA = sampler.getSample().staticSample + pointB = sampler.getSample().staticSample assert pointA != pointB assert space.distance(pointA, pointA) == 0 assert space.distance(pointB, pointB) == 0 diff --git a/tests/test_grid.py b/tests/test_grid.py index de16685..0378a9e 100644 --- a/tests/test_grid.py +++ b/tests/test_grid.py @@ -14,7 +14,9 @@ def test_grid(): dict_samples = defaultdict(int) while True: try: - sample = sampler.nextSample() + sample = sampler.getSample() + sample.update(None) + sample = sample.staticSample dict_samples[(sample.weather[0], sample.car_positions[0], sample.car_positions[1])] = 0 except TerminationException: @@ -46,7 +48,9 @@ def f(sample): y_samples = [] for i in range(21): - sample = sampler.nextSample() + sample = sampler.getSample() + sample.update(None) + sample = sample.staticSample samples.append(sample) y_samples.append(f(sample)) @@ -59,6 +63,6 @@ def test_grid_non_standardizable(): 'b': Feature(FilteredDomain(Box([0,1]), lambda x: x[0] > 0.5)) }) sampler = FeatureSampler.gridSamplerFor(space) - samples = list(sampler) + samples = [s.staticSample for s in sampler] assert len(samples) == 13 assert all(sample.b[0] > 0.5 for sample in samples) diff --git a/tests/test_halton.py b/tests/test_halton.py index 4c3cb7f..6981fd8 100644 --- a/tests/test_halton.py +++ b/tests/test_halton.py @@ -24,7 +24,7 @@ def test_halton(): for i in range(3): print(f'Sample #{i}:') - print(sampler.nextSample()) + print(sampler.getSample().staticSample) def test_save_restore(tmpdir): space = FeatureSpace({ diff --git a/tests/test_samplers.py b/tests/test_samplers.py index 82e4a4c..ec8c56b 100644 --- a/tests/test_samplers.py +++ b/tests/test_samplers.py @@ -16,8 +16,8 @@ def test_feature_sampling(): timeBound=10) sampler = FeatureSampler.randomSamplerFor(space) - static_sample, dynamic_points_gen = sampler.getSample() - static_point = static_sample[0] + sample = sampler.getSample() + static_point = sample.staticSample static_dict = static_point._asdict() assert "a" in static_dict @@ -31,8 +31,7 @@ def test_feature_sampling(): assert all(0 <= v[0] <= 1 for v in static_dict["b"]) for _ in range(space.timeBound): - dynamic_point = next(dynamic_points_gen) - print(dynamic_point) + dynamic_point = sample.getDynamicSample() dynamic_dict = dynamic_point._asdict() assert "a" not in dynamic_dict assert "b" not in dynamic_dict @@ -74,7 +73,7 @@ def check(samples): assert any(sample[0][0].position[0] < sample[1][1].position[0] for sample in samples) - check([sampler.nextSample() for i in range(100)]) + check([sampler.getSample()[0] for _ in range(100)]) check(list(itertools.islice(sampler, 100))) def test_space_random(): @@ -86,7 +85,6 @@ def test_space_random(): def check(samples): for sample in samples: - assert type(sample) is space.makePoint a = sample.a assert type(a) is tuple assert len(a) == 1 @@ -110,8 +108,8 @@ def check(samples): assert any(len(sample.b) == 1 for sample in samples) assert any(len(sample.b) == 2 for sample in samples) - check([sampler.nextSample() for i in range(100)]) - check(list(itertools.islice(sampler, 100))) + check([sampler.getSample().staticSample for i in range(100)]) + check(list(s.staticSample for s in itertools.islice(sampler, 100))) def test_random_restore(tmpdir): space = FeatureSpace({ @@ -122,7 +120,7 @@ def test_random_restore(tmpdir): path = os.path.join(tmpdir, 'blah.dat') sampler.saveToFile(path) - sample1 = sampler.nextSample() + sample1 = sampler.getSample() sampler = FeatureSampler.restoreFromFile(path) - sample2 = sampler.nextSample() - assert sample1 == sample2 + sample2 = sampler.getSample() + assert sample1.staticSample == sample2.staticSample diff --git a/tests/test_simulatedAnnealing.py b/tests/test_simulatedAnnealing.py index 61ade37..562d09d 100644 --- a/tests/test_simulatedAnnealing.py +++ b/tests/test_simulatedAnnealing.py @@ -45,7 +45,7 @@ def test_simulatedAnnealing(): }) def f(sample): - return sample.cars[0].heading[0] - 0.75 + return sample.staticSample.cars[0].heading[0] - 0.75 sampler = FeatureSampler.simulatedAnnealingSamplerFor(space, sa_params) diff --git a/tests/utils.py b/tests/utils.py index 5b5d76f..063814c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,11 +4,11 @@ from verifai.samplers import FeatureSampler def sampleWithFeedback(sampler, num_samples, f): - feedback = None samples = [] for i in range(num_samples): - sample = sampler.nextSample(feedback) + sample = sampler.getSample() feedback = f(sample) + sample.update(feedback) print(f'Sample #{i}:') print(sample) samples.append(sample) @@ -19,18 +19,27 @@ def checkSaveRestore(sampler, tmpdir, iterations=1): feedback = None for i in range(iterations): sampler.saveToFile(path) - sample1 = sampler.nextSample(feedback) - sample2 = sampler.nextSample(-1) + # sample1 = sampler.nextSample(feedback) + # sample2 = sampler.nextSample(-1) + sample1 = sampler.getSample() + sample1.update(-1) + sample2 = sampler.getSample() sampler = FeatureSampler.restoreFromFile(path) - sample1b = sampler.nextSample(feedback) - sample2b = sampler.nextSample(-1) - assert sample1 != sample2 - assert sample1 == sample1b - assert sample2 == sample2b + # sample1b = sampler.nextSample(feedback) + # sample2b = sampler.nextSample(-1) + sample1b = sampler.getSample() + sample1b.update(-1) + sample2b = sampler.getSample() + sample2b.update(1) + assert sample1.staticSample != sample2.staticSample + assert sample1.staticSample == sample1b.staticSample + assert sample2.staticSample == sample2b.staticSample sampler.saveToFile(path) - sample3 = sampler.nextSample(1) + # sample3 = sampler.nextSample(1) + sample3 = sampler.getSample() sampler = FeatureSampler.restoreFromFile(path) - sample3b = sampler.nextSample(1) - assert sample3 not in (sample1, sample2) - assert sample3 == sample3b - feedback = 1 + sample3b = sampler.getSample() + assert sample3.staticSample not in (sample1.staticSample, sample2.staticSample) + assert sample3.staticSample == sample3b.staticSample + # feedback = 1 + sample3b.update(1) From c07dc0f8de3130e86f2d8ef591efc789f3fafdf4 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Tue, 25 Nov 2025 12:38:15 -0800 Subject: [PATCH 04/11] Modified test to ensure time series value stability. --- tests/scenic/scenic_driving_behavior.scenic | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/scenic/scenic_driving_behavior.scenic b/tests/scenic/scenic_driving_behavior.scenic index ab19e90..b4acb61 100644 --- a/tests/scenic/scenic_driving_behavior.scenic +++ b/tests/scenic/scenic_driving_behavior.scenic @@ -8,6 +8,9 @@ foo = VerifaiRange(0,0.01, timeSeries=True) behavior TestBehavior(): while True: + val1 = foo.getSample() + val2 = foo.getSample() + assert val1 == val2 take SetThrottleAction(foo.getSample()) ego = new Car on road, with behavior TestBehavior() From 0787c9347b4052a171fc6b1aa2f976174710c84c Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Tue, 25 Nov 2025 12:55:10 -0800 Subject: [PATCH 05/11] Clarified double sampling test --- tests/scenic/scenic_driving_behavior.scenic | 3 -- tests/scenic/test_scenic.py | 32 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/scenic/scenic_driving_behavior.scenic b/tests/scenic/scenic_driving_behavior.scenic index b4acb61..ab19e90 100644 --- a/tests/scenic/scenic_driving_behavior.scenic +++ b/tests/scenic/scenic_driving_behavior.scenic @@ -8,9 +8,6 @@ foo = VerifaiRange(0,0.01, timeSeries=True) behavior TestBehavior(): while True: - val1 = foo.getSample() - val2 = foo.getSample() - assert val1 == val2 take SetThrottleAction(foo.getSample()) ego = new Car on road, with behavior TestBehavior() diff --git a/tests/scenic/test_scenic.py b/tests/scenic/test_scenic.py index 30efc8e..a0041a8 100644 --- a/tests/scenic/test_scenic.py +++ b/tests/scenic/test_scenic.py @@ -199,3 +199,35 @@ def test_driving_dynamic_behavior(pathToLocalFile): server_class=ScenicServer, server_options=server_options) falsifier.run_falsifier() + +double_access_scenario = """ +model scenic.simulators.newtonian.model +param verifaiTimeBound = 100 +foo = VerifaiRange(0, 0.01, timeSeries=True) +behavior TestBehavior(): + while True: + foo.getSample() + foo.getSample() + wait +ego = new Object with behavior TestBehavior() +""" + +def test_double_time_series_access(): + with pytest.raises(RuntimeError): + sampler = ScenicSampler.fromScenicCode( + double_access_scenario, + model='scenic.simulators.newtonian.model', + maxIterations=1, + params=dict(render=False), + ) + falsifier_params = DotMap( + n_iters=3, + save_error_table=False, + save_safe_table=False, + ) + server_options = DotMap(maxSteps=2, verbosity=3) + falsifier = generic_falsifier(sampler=sampler, + falsifier_params=falsifier_params, + server_class=ScenicServer, + server_options=server_options) + falsifier.run_falsifier() From 0b0477fa374be83ffd2736a73bb58df93c8b6623 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Wed, 26 Nov 2025 21:23:39 -0800 Subject: [PATCH 06/11] Banned dynamic sampling from generic server. --- src/verifai/server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/verifai/server.py b/src/verifai/server.py index 32e5f64..5539386 100644 --- a/src/verifai/server.py +++ b/src/verifai/server.py @@ -144,6 +144,8 @@ def __init__(self, sampling_data, monitor, options={}): sampler_params=params ) + if (self.sample_space.dynamicFeatureNamed) != 0: + raise ValueError("Sample space for `Server` cannot contain `TimeSeriesFeature`") def listen(self): client_socket, addr = self.socket.accept() @@ -195,7 +197,7 @@ def run_server(self): start = time.time() sample = self.get_sample() after_sampling = time.time() - self.lastValue = self.evaluate_sample(sample) + self.lastValue = self.evaluate_sample(sample.staticSample) sample.update(self.lastValue) after_simulation = time.time() timings = ServerTimings(sample_time=(after_sampling - start), From 2e266d9e7b1673b368c7cfa499cd52d8d6d0d2ff Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Tue, 2 Dec 2025 14:47:39 -0800 Subject: [PATCH 07/11] Dynamic sampling tweaks. --- src/verifai/samplers/scenic_sampler.py | 14 ++++++++++++-- tests/scenic/scenic_driving_behavior.scenic | 3 +-- tests/scenic/test_scenic.py | 5 +++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/verifai/samplers/scenic_sampler.py b/src/verifai/samplers/scenic_sampler.py index 45e4a11..6e39526 100644 --- a/src/verifai/samplers/scenic_sampler.py +++ b/src/verifai/samplers/scenic_sampler.py @@ -261,7 +261,7 @@ def __init__(self, scenario, maxIterations=None, ignoredProperties=None): super().__init__(space) @classmethod - def fromScenario(cls, path, maxIterations=None, + def fromScenario(cls, path, maxIterations=None, maxSteps=None, ignoredProperties=None, **kwargs): """Create a sampler corresponding to a Scenic program. @@ -280,14 +280,24 @@ def fromScenario(cls, path, maxIterations=None, e.g. ``params`` to override global parameters or ``model`` to set the :term:`world model`. """ + if "params" not in kwargs: + kwargs["params"] = {} + + kwargs["params"]["timeBound"] = maxSteps if maxSteps else 0 + scenario = scenic.scenarioFromFile(path, **kwargs) return cls(scenario, maxIterations=maxIterations, ignoredProperties=ignoredProperties) @classmethod - def fromScenicCode(cls, code, maxIterations=None, + def fromScenicCode(cls, code, maxIterations=None, maxSteps=None, ignoredProperties=None, **kwargs): """As above, but given a Scenic program as a string.""" + if "params" not in kwargs: + kwargs["params"] = {} + + kwargs["params"]["timeBound"] = maxSteps if maxSteps else 0 + scenario = scenic.scenarioFromString(code, **kwargs) return cls(scenario, maxIterations=maxIterations, ignoredProperties=ignoredProperties) diff --git a/tests/scenic/scenic_driving_behavior.scenic b/tests/scenic/scenic_driving_behavior.scenic index ab19e90..9240a34 100644 --- a/tests/scenic/scenic_driving_behavior.scenic +++ b/tests/scenic/scenic_driving_behavior.scenic @@ -1,10 +1,9 @@ param map = localPath('Town01.xodr') param carla_map = 'Town01' -param verifaiTimeBound = 100 model scenic.domains.driving.model -foo = VerifaiRange(0,0.01, timeSeries=True) +foo = TimeSeries(VerifaiRange(0,0.01)) behavior TestBehavior(): while True: diff --git a/tests/scenic/test_scenic.py b/tests/scenic/test_scenic.py index a0041a8..951e184 100644 --- a/tests/scenic/test_scenic.py +++ b/tests/scenic/test_scenic.py @@ -187,6 +187,7 @@ def test_driving_dynamic_behavior(pathToLocalFile): model='scenic.simulators.newtonian.driving_model', params=dict(render=False), mode2D=True, + maxSteps=2 ) falsifier_params = DotMap( n_iters=3, @@ -202,8 +203,7 @@ def test_driving_dynamic_behavior(pathToLocalFile): double_access_scenario = """ model scenic.simulators.newtonian.model -param verifaiTimeBound = 100 -foo = VerifaiRange(0, 0.01, timeSeries=True) +foo = TimeSeries(VerifaiRange(0, 0.01)) behavior TestBehavior(): while True: foo.getSample() @@ -218,6 +218,7 @@ def test_double_time_series_access(): double_access_scenario, model='scenic.simulators.newtonian.model', maxIterations=1, + maxSteps=2, params=dict(render=False), ) falsifier_params = DotMap( From 9c5539b2794f5a18fbbda400d4ee1bb095e9bce9 Mon Sep 17 00:00:00 2001 From: Eric Vin <8935814+Eric-Vin@users.noreply.github.com> Date: Wed, 17 Dec 2025 19:36:30 -0800 Subject: [PATCH 08/11] Apply suggestions from code review Co-authored-by: Daniel Fremont --- src/verifai/features/features.py | 2 +- src/verifai/samplers/feature_sampler.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/verifai/features/features.py b/src/verifai/features/features.py index 49ab6d5..e695029 100644 --- a/src/verifai/features/features.py +++ b/src/verifai/features/features.py @@ -1042,7 +1042,7 @@ def domains(self): corresponding Domain of other features. If the FeatureSpace has no feature lists, then returns (None, dom) where dom is the fixed Domain of all features. If any Features are TimeSeriesFeatures then they are - expanded to the a max of timeBound. + expanded to a max of timeBound. """ fixedDomains = {} lengthDomains = {} diff --git a/src/verifai/samplers/feature_sampler.py b/src/verifai/samplers/feature_sampler.py index e79fd69..9a14190 100644 --- a/src/verifai/samplers/feature_sampler.py +++ b/src/verifai/samplers/feature_sampler.py @@ -300,10 +300,7 @@ def getSample(self): if not self.space.featureNamed[f].lengthDomain: point_dict[f] = val[t] else: - feat_list = [] - for l in range(len(val)): - feat_list.append(val[l][t]) - point_dict[f] = tuple(feat_list) + point_dict[f] = tuple(v[t] for v in val) dynamic_points.append(self.space.makeDynamicPoint(*point_dict.values())) From 62c97f221b7cafbf5673f6c5beeaad5de5608c37 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Wed, 24 Dec 2025 14:28:02 -0800 Subject: [PATCH 09/11] Progress on PR feedback --- src/verifai/features/features.py | 287 +++++++++++++++----- src/verifai/samplers/domain_sampler.py | 12 - src/verifai/samplers/feature_sampler.py | 80 ++---- src/verifai/samplers/scenic_sampler.py | 26 +- src/verifai/server.py | 2 +- tests/scenic/scenic_driving_behavior.scenic | 6 +- tests/test_features.py | 65 ++++- tests/test_samplers.py | 27 +- tests/utils.py | 6 - 9 files changed, 310 insertions(+), 201 deletions(-) diff --git a/src/verifai/features/features.py b/src/verifai/features/features.py index e695029..ad2de4d 100644 --- a/src/verifai/features/features.py +++ b/src/verifai/features/features.py @@ -9,6 +9,7 @@ import random import itertools import functools +from abc import ABC, abstractmethod from collections import OrderedDict, namedtuple import numpy as np @@ -970,7 +971,7 @@ def _timeExpandDomain(domain, timeBound): def fixedDomains(self, timeBound): """Return the fixed-length Domains associated with this feature.""" - timeExpandedDomain = self._timeExpandDomain(self.domain, timeBound) + timeExpandedDomain = self._timeExpandDomain(self.domain, timeBound) if timeBound is not None else self.domain if not self.lengthDomain: domains = timeExpandedDomain @@ -995,15 +996,110 @@ def __repr__(self): return rep + ')' class TimeSeriesFeature(Feature): + """A feature with a value at each timesetep of a simulation.""" @staticmethod def _timeExpandDomain(domain, timeBound): return Array(domain, (timeBound,)) -### Feature spaces +### Feature spaces and Samples + +class Sample(ABC): + """A sample from a feature space, containing static points and able to generate dynamic points. + + Args: + space (FeatureSpace): The feature space this Sample was sampled from. + dynamicSampleLengths (dict): A dictionary containing the lengths of each dynamic feature + with a length domain. + """ + def __init__(self, space, dynamicSampleLengths): + self.space = space + self.dynamicSampleHistory = [] + self.dynamicSampleLengths = dynamicSampleLengths + + @property + @abstractmethod + def staticSample(self): + pass + + @abstractmethod + def _getDynamicSample(self, info): + pass + + def getDynamicSample(self, info=None): + sample = self._getDynamicSample(info) + self.dynamicSampleHistory.append(sample) + return sample + + @abstractmethod + def update(self, rho): + pass + + def __getattr__(self, attr): + space = super().__getattribute__("space") + if attr in space.staticFeatureNamed: + return getattr(space.staticSample, attr) + elif attr in space.dynamicFeatureNamed: + class DynamicFeatureHelper: + def __init__(self, dynamicSampleHistory, attr): + self.dynamicSampleHistory = dynamicSampleHistory + self.attr = attr + + def __getitem__(self, i): + return getattr(self.dynamicSampleHistory[i], self.attr) + + return DynamicFeatureHelper(self.dynamicSampleHistory, attr) + else: + return super().__getattr__(attr) + + +class CompleteSample(Sample): + """A completed sample, which has fully computed static and dynamic points. + + Args: + space (FeatureSpace): The feature space this Sample was sampled from. + staticSample: The static point for this sample + dynamicSampleList: A list of the dynamic points for this sample. + updateCallback: A callback that is called with the value passed to update. + dynamicSampleLengths (dict): A dictionary containing the lengths of each dynamic feature + with a length domain. + """ + def __init__(self, space, staticSample, dynamicSampleList, updateCallback, dynamicSampleLengths): + super().__init__(space, dynamicSampleLengths) + self._staticSample = staticSample + self._dynamicSampleList = dynamicSampleList + self._updateCallback = updateCallback + self._i = 0 + + @property + def staticSample(self): + return self._staticSample + + def _getDynamicSample(self, info): + if self.space.timeBound == 0: + raise RuntimeError("Called `getDynamicSample` with `timeBound` of `FeatureSpace` set to 0") + + if self._i >= self.space.timeBound: + raise RuntimeError("Exceeded `timeBound` of `FeatureSpace`") + + assert self._i < len(self._dynamicSampleList) + + dynamic_sample = self._dynamicSampleList[self._i] + self._i += 1 + + return dynamic_sample + + def update(self, rho): + return self._updateCallback(rho) class FeatureSpace: """A space consisting of named features. + Args: + features (Iterable): An iterable of the `Feature` objects in this space. + distanceMetric (function; optional): An optional distance metric to be used with this space. + timeBound (int; optional): An upper bound on the number of timesteps of a simulation using + this space. + .. testcode:: FeatureSpace({ @@ -1023,7 +1119,8 @@ def __init__(self, features, distanceMetric=None, timeBound=0): self.dynamicFeatureNamed = OrderedDict({name: feat for name, feat in self.featureNamed.items() if isinstance(feat, TimeSeriesFeature)}) - # self.makePoint = namedtuple('SpacePoint', self.featureNamed) + self.hasTimeSeries = len(self.dynamicFeatureNamed) > 0 + self.makeStaticPoint = namedtuple('StaticSpacePoint', self.staticFeatureNamed) self.makeDynamicPoint = namedtuple('DynamicSpacePoint', self.dynamicFeatureNamed) @@ -1085,7 +1182,6 @@ def flatten(self, point, fixedDimension=False): lists had their maximum lengths and time steps, with None as a placeholder. This means that all points in the space will flatten to the same length. """ - from verifai.samplers.feature_sampler import Sample assert isinstance(point, Sample) flattened = [] @@ -1094,7 +1190,7 @@ def flatten(self, point, fixedDimension=False): if feature.lengthDomain: length = len(value) flattened.append(length) - fixedDomain = feature.fixedDomains(self.timeBound)[length] + fixedDomain = feature.fixedDomains(None)[length] fixedDomain.flattenOnto(value, flattened) if fixedDimension: # add padding to maximum length sizePerElt = domain.flattenedDimension @@ -1104,34 +1200,38 @@ def flatten(self, point, fixedDimension=False): else: domain.flattenOnto(value, flattened) - flattened.append(len(point.dynamicSampleHistory)) - - for feature_i, feature in enumerate(self.dynamicFeatureNamed.values()): - if feature.lengthDomain: - length = len(value) - else: - length = None + if self.hasTimeSeries: + duration = len(point.dynamicSampleHistory) + flattened.append(duration) - flattened.append(length) - - for dynamic_point in point.dynamicSampleHistory: - value = dynamic_point[feature_i] + for feature_i, f in enumerate(self.dynamicFeatureNamed.items()): + feature_name, feature = f domain = feature.domain + sizePerElt = domain.flattenedDimension - if length is None: - domain.flattenOnto(value, flattened) + if feature.lengthDomain: + length = point.dynamicSampleLengths[feature_name] else: - fixedDomain = feature.fixedDomains(self.timeBound)[length] - fixedDomain.flattenOnto(value, flattened) - if fixedDimension: - sizePerElt = domain.flattenedDimension - needed = (feature.maxLength - length) * sizePerElt - for i in range(needed): - flattened.append(None) - - if fixedDimension: - needed = (self.space.timeBound - len(point.dynamicSampleHistory)) * length * sizePerElt - flattened += [None for _ in range(needed)] + length = None + + flattened.append(length) + + for dynamic_point in point.dynamicSampleHistory: + value = dynamic_point[feature_i] + + if length is None: + domain.flattenOnto(value, flattened) + else: + fixedDomain = feature.fixedDomains(None)[length] + fixedDomain.flattenOnto(value, flattened) + if fixedDimension: + needed = (feature.maxLength - length) * sizePerElt + for i in range(needed): + flattened.append(None) + + if fixedDimension: + needed = (self.timeBound - len(point.dynamicSampleHistory)) * feature.maxLength * sizePerElt + flattened += [None for _ in range(needed)] flattened_point = tuple(flattened) if fixedDimension: @@ -1145,7 +1245,8 @@ def fixedFlattenedDimension(self): Also an upper bound on the length of the vector returned by flatten by default, when fixedDimension=False.""" dim = 0 - dim += 1 # Timesteps + if self.hasTimeSeries: + dim += 1 # Timesteps for feature in self.features: domain = feature.domain timeMult = self.timeBound if isinstance(feature, TimeSeriesFeature) else 1 @@ -1191,12 +1292,12 @@ def meaningOfFlatCoordinate(self, index, pointName='point'): for name, feature in self.dynamicFeatureNamed.items(): domain = feature.domain - for time_i in range(self.timeBound): - if feature.lengthDomain: - if index == 0: - return f'len({pointName}.dynamicSampleHistory[{time_i}].{name})' - else: - index -= 1 + if feature.lengthDomain: + if index == 0: + return f'{pointName}.dynamicSampleLengths["{name}"]' + else: + index -= 1 + for time_i in range(self.timeBound): elem = index // domain.flattenedDimension if elem < feature.maxLength: subIndex = index % domain.flattenedDimension @@ -1204,7 +1305,9 @@ def meaningOfFlatCoordinate(self, index, pointName='point'): return domain.meaningOfFlatCoordinate(subIndex, pointName=subPoint) index -= feature.maxLength * domain.flattenedDimension - else: + else: + index -= 1 + for time_i in range(self.timeBound): if index < domain.flattenedDimension: subPoint = f'{pointName}.dynamicSampleHistory[{time_i}].{name}' return domain.meaningOfFlatCoordinate(index, @@ -1218,9 +1321,8 @@ def pandasIndexForFlatCoordinate(self, index): See meaningOfFlatCoordinate, and Domain.pandasIndexForFlatCoordinate. """ - #TODO Update assert 0 <= index < self.fixedFlattenedDimension - for name, feature in self.namedFeatures: + for name, feature in self.staticFeatureNamed.items(): domain = feature.domain if feature.lengthDomain: if index == 0: @@ -1238,8 +1340,35 @@ def pandasIndexForFlatCoordinate(self, index): panda = domain.pandasIndexForFlatCoordinate(index) return (name,) + panda index -= domain.flattenedDimension + + if index == 0: + return ("dynamicSampleHistory", "length") + index -= 1 + + for name, feature in self.dynamicFeatureNamed.items(): + domain = feature.domain + if feature.lengthDomain: + if index == 0: + return (name, 'length') + else: + index -= 1 + for time_i in range(self.timeBound): + elem = index // domain.flattenedDimension + if elem < feature.maxLength: + subIndex = index % domain.flattenedDimension + panda = domain.pandasIndexForFlatCoordinate(subIndex) + return (name, elem) + panda + index -= feature.maxLength * domain.flattenedDimension + else: + for time_i in range(self.timeBound): + if index < domain.flattenedDimension: + panda = domain.pandasIndexForFlatCoordinate(index) + return (name,) + panda + index -= domain.flattenedDimension + raise RuntimeError('impossible index arithmetic') + def coordinateIsNumerical(self, index): """Whether the value of a coordinate is intrinsically numerical. @@ -1269,18 +1398,19 @@ def coordinateIsNumerical(self, index): for name, feature in self.dynamicFeatureNamed.items(): domain = feature.domain - for time_i in range(self.timeBound): - if feature.lengthDomain: - if index == 0: - return True - else: - index -= 1 + if feature.lengthDomain: + if index == 0: + return True + else: + index -= 1 + for time_i in range(self.timeBound): elem = index // domain.flattenedDimension if elem < feature.maxLength: subIndex = index % domain.flattenedDimension return domain.coordinateIsNumerical(subIndex) index -= feature.maxLength * domain.flattenedDimension - else: + else: + for time_i in range(self.timeBound): if index < domain.flattenedDimension: return domain.coordinateIsNumerical(index) index -= domain.flattenedDimension @@ -1289,8 +1419,6 @@ def coordinateIsNumerical(self, index): def unflatten(self, coords, fixedDimension=False): """Unflatten a tuple of coordinates to a point in this space.""" - from verifai.samplers.feature_sampler import LateFeatureSample - staticValues = [] iterator = iter(coords) @@ -1298,7 +1426,7 @@ def unflatten(self, coords, fixedDimension=False): domain = feature.domain if feature.lengthDomain: length = next(iterator) - fixedDomain = feature.fixedDomains(self.timeBound)[length] + fixedDomain = feature.fixedDomains(None)[length] staticValues.append(fixedDomain.unflattenIterator(iterator)) if fixedDimension: # consume padding sizePerElt = domain.flattenedDimension @@ -1310,41 +1438,52 @@ def unflatten(self, coords, fixedDimension=False): staticSample = self.makeStaticPoint(*staticValues) - timeSteps = next(iterator) + if self.hasTimeSeries: + duration = next(iterator) - dynamicValuesList = [[] for _ in range(timeSteps)] - - for feature in self.dynamicFeatureNamed.values(): - domain = feature.domain - length = next(iterator) - - for time_i in range(timeSteps): + dynamicValuesList = [[] for _ in range(duration)] + dynamicSampleLengths = {} + for feature_name, feature in self.dynamicFeatureNamed.items(): + domain = feature.domain + sizePerElt = domain.flattenedDimension + length = next(iterator) + dynamicSampleLengths[feature_name] = length + + for time_i in range(duration): + if length is None: + dynamicValuesList[time_i].append(domain.unflattenIterator(iterator)) + else: + fixedDomain = feature.fixedDomains(None)[length] + dynamicValuesList[time_i].append(fixedDomain.unflattenIterator(iterator)) + if fixedDimension: # consume padding + needed = (feature.maxLength - length) * sizePerElt + for _ in range(needed): + next(iterator) + + if fixedDimension: + needed = (self.timeBound - duration) * length * sizePerElt + for _ in range(needed): + next(iterator) - if length is None: - dynamicValuesList[time_i].append(domain.unflattenIterator(iterator)) - else: - fixedDomain = feature.fixedDomains(self.timeBound)[length] - dynamicValuesList[time_i].append(fixedDomain.unflattenIterator(iterator)) - if fixedDimension: # consume padding - sizePerElt = domain.flattenedDimension - needed = (feature.maxLength - length) * sizePerElt - for i in range(needed): - next(iterator) + dynamicSampleList = [self.makeDynamicPoint(*dynamicValues) for dynamicValues in dynamicValuesList] + else: + duration = 0 + dynamicSampleList = [] + dynamicSampleLengths = {} - if fixedDimension: - needed = (self.space.timeBound - timeSteps) * length * sizePerElt - for i in range(needed): - next(iterator) + updateCallback = lambda rho: None - dynamicSampleList = [self.makeDynamicPoint(*dynamicValues) for dynamicValues in dynamicValuesList] + sample = CompleteSample(space=self, staticSample=staticSample, dynamicSampleList=dynamicSampleList, updateCallback=updateCallback,dynamicSampleLengths=dynamicSampleLengths) - updateCallback = lambda rho: None + for _ in range(duration): + sample.getDynamicSample() - return LateFeatureSample(space=None, staticSample=staticSample, dynamicSampleList=dynamicSampleList, updateCallback=updateCallback) + return sample def __repr__(self): rep = f'FeatureSpace({self.featureNamed}' if self.distanceMetric is not None: rep += f', distanceMetric={self.distanceMetric}' return rep + ')' + diff --git a/src/verifai/samplers/domain_sampler.py b/src/verifai/samplers/domain_sampler.py index 83426ae..a1ba7b2 100644 --- a/src/verifai/samplers/domain_sampler.py +++ b/src/verifai/samplers/domain_sampler.py @@ -47,18 +47,6 @@ def update(self, sample, info, rho): """ pass - # TODO: Deprecate - # def nextSample(self, feedback=None): - # """Generate the next sample, given feedback from the last sample. - - # This exists only for backwards-compatibility. It has been replaced by - # the getSample and update APIs. - # """ - # if self.last_sample is not None: - # self.update(self.last_sample, self.last_info, feedback) - # self.last_sample, self.last_info = self.getSample() - # return self.last_sample - def __iter__(self): try: while True: diff --git a/src/verifai/samplers/feature_sampler.py b/src/verifai/samplers/feature_sampler.py index 9a14190..c3e42ca 100644 --- a/src/verifai/samplers/feature_sampler.py +++ b/src/verifai/samplers/feature_sampler.py @@ -11,7 +11,7 @@ from abc import ABC, abstractmethod from contextlib import contextmanager -from verifai.features import FilteredDomain, TimeSeriesFeature +from verifai.features import FilteredDomain, TimeSeriesFeature, Sample, CompleteSample from verifai.samplers.domain_sampler import SplitSampler, TerminationException from verifai.samplers.rejection import RejectionSampler from verifai.samplers.halton import HaltonSampler @@ -23,29 +23,6 @@ from verifai.samplers.simulated_annealing import SimulatedAnnealingSampler from verifai.samplers.grid_sampler import GridSampler -class Sample(ABC): - def __init__(self, space): - self.space = space - self.dynamicSampleHistory = [] - - @property - @abstractmethod - def staticSample(self): - pass - - @abstractmethod - def _getDynamicSample(self, info): - pass - - def getDynamicSample(self, info=None): - sample = self._getDynamicSample(info) - self.dynamicSampleHistory.append(sample) - return sample - - @abstractmethod - def update(self, rho): - pass - ### Samplers defined over FeatureSpaces class FeatureSampler(ABC): @@ -211,38 +188,15 @@ def __iter__(self): except TerminationException: return -class LateFeatureSample(Sample): - def __init__(self, space, staticSample, dynamicSampleList, updateCallback): - super().__init__(space) - self._staticSample = staticSample - self._dynamicSampleList = dynamicSampleList - self._updateCallback = updateCallback - self._i = 0 - - @property - def staticSample(self): - return self._staticSample - - def _getDynamicSample(self, info): - if self.space.timeBound == 0: - raise RuntimeError("Called `getDynamicSample` with `timeBound` of `FeatureSpace` set to 0") - - if self._i >= self.space.timeBound: - raise RuntimeError("Exceeded `timeBound` of `FeatureSpace`") - - assert self._i < len(self._dynamicSampleList) - - dynamic_sample = self._dynamicSampleList[self._i] - self._i += 1 - - return dynamic_sample - - def update(self, rho): - return self._updateCallback(rho) class LateFeatureSampler(FeatureSampler): - """FeatureSampler that works by first sampling only lengths of feature - lists, then sampling from the resulting fixed-dimensional Domain. + """ FeatureSampler that greedily samples a CompleteSample. + + FeatureSampler works as follows: + 1. Sample lengths of feature lists. + 2. Expand TimeSeriesFeatures into flattened features with of length + space.timeBound. + 2. Sample from the resulting fixed-dimensional Domains. e.g. LateFeatureSampler(space, RandomSampler, HaltonSampler) creates a FeatureSampler which picks lengths uniformly at random and applies @@ -264,12 +218,12 @@ def __init__(self, space, makeLengthSampler, makeDomainSampler): for point, domain in fixedDomains.items() } - self.id_metadata_dict = {} + self._id_metadata_dict = {} self._last_id = 0 - def get_info_id(self, info, length, sample): + def _get_info_id(self, info, length, sample): self._last_id += 1 - self.id_metadata_dict[self._last_id] = (info, length, sample) + self._id_metadata_dict[self._last_id] = (info, length, sample) return self._last_id def getSample(self): @@ -281,7 +235,7 @@ def getSample(self): domainPoint, info2 = self.domainSamplers[length].getSample() info = (info1, info2) - sample_id = self.get_info_id(info, length, domainPoint) + sample_id = self._get_info_id(info, length, domainPoint) update_callback = lambda rho: self.update(sample_id, rho) # Make static points and iterable over dynamic points @@ -304,10 +258,16 @@ def getSample(self): dynamic_points.append(self.space.makeDynamicPoint(*point_dict.values())) - return LateFeatureSample(self.space, static_point, dynamic_points, update_callback) + + dynamicSampleLengths = ({feature_name: getattr(length, feature_name)[0] + for feature_name, feature in self.space.dynamicFeatureNamed.items() + if feature.lengthDomain} + if self.lengthSampler else {}) + + return CompleteSample(self.space, static_point, dynamic_points, update_callback, dynamicSampleLengths) def update(self, sample_id, rho): - info, lengthPoint, domainPoint = self.id_metadata_dict[sample_id] + info, lengthPoint, domainPoint = self._id_metadata_dict[sample_id] if self.lengthSampler is None: self.domainSamplers[None].update(domainPoint, info[1], rho) diff --git a/src/verifai/samplers/scenic_sampler.py b/src/verifai/samplers/scenic_sampler.py index 6e39526..d919182 100644 --- a/src/verifai/samplers/scenic_sampler.py +++ b/src/verifai/samplers/scenic_sampler.py @@ -224,8 +224,8 @@ def spaceForScenario(scenario, ignoredProperties): return space, quotedParams class ScenicSample(Sample): - def __init__(self, space, staticSample, updateCallback): - super().__init__(space) + def __init__(self, space, staticSample, updateCallback, dynamicSampleLengths): + super().__init__(space, dynamicSampleLengths) self._staticSample = staticSample self._updateCallback = updateCallback @@ -234,7 +234,7 @@ def staticSample(self): return self._staticSample def _getDynamicSample(self, info): - raise RuntimeError("ScenicSample does not support dynamic sampling.") + raise RuntimeError("ScenicSampler does not support dynamic sampling.") def update(self, rho): self._updateCallback(rho) @@ -310,10 +310,7 @@ def getSample(self): self.lastFeedback = None self.lastScene, _ = ret - staticSample = self.pointForScene(self.lastScene) - updateCallback = lambda rho: self.update(0, rho) - - return ScenicSample(self.space, staticSample, updateCallback) + return self.pointForScene(self.lastScene) def update(self, sample_id, rho): assert sample_id == 0 @@ -321,14 +318,6 @@ def update(self, sample_id, rho): raise RuntimeError("Called `update` twice in a row (ScenicSampler does not support non-sequential sampling)") self.lastFeedback = rho - # TODO: Deprecate - # def nextSample(self, feedback=None): - # ret = self.scenario.generate( - # maxIterations=self.maxIterations, feedback=feedback, verbosity=0 - # ) - # self.lastScene, _ = ret - # return self.pointForScene(self.lastScene) - def pointForScene(self, scene): """Convert a sampled Scenic :obj:`~scenic.core.scenarios.Scene` to a point in our feature space. @@ -362,7 +351,12 @@ def pointForScene(self, scene): params[param] = pointForValue(subdom, scene.params[originalName]) paramPoint = paramDomain.makePoint(**params) - return self.space.makeStaticPoint(objects=objPoint, params=paramPoint) + staticSample = self.space.makeStaticPoint(objects=objPoint, params=paramPoint) + + updateCallback = lambda rho: self.update(0, rho) + dynamicSampleLengths = [] + + return ScenicSample(self.space, staticSample, updateCallback, dynamicSampleLengths) @staticmethod def nameForObject(i): diff --git a/src/verifai/server.py b/src/verifai/server.py index 5539386..87a75fa 100644 --- a/src/verifai/server.py +++ b/src/verifai/server.py @@ -144,7 +144,7 @@ def __init__(self, sampling_data, monitor, options={}): sampler_params=params ) - if (self.sample_space.dynamicFeatureNamed) != 0: + if self.sample_space.hasTimeSeries: raise ValueError("Sample space for `Server` cannot contain `TimeSeriesFeature`") def listen(self): diff --git a/tests/scenic/scenic_driving_behavior.scenic b/tests/scenic/scenic_driving_behavior.scenic index 9240a34..20c0e67 100644 --- a/tests/scenic/scenic_driving_behavior.scenic +++ b/tests/scenic/scenic_driving_behavior.scenic @@ -6,8 +6,12 @@ model scenic.domains.driving.model foo = TimeSeries(VerifaiRange(0,0.01)) behavior TestBehavior(): + lastVal = None while True: - take SetThrottleAction(foo.getSample()) + newVal = foo.getSample() + assert newVal != lastVal, (newVal, lastVal) + lastVal = newVal + take SetThrottleAction(newVal) ego = new Car on road, with behavior TestBehavior() new Car behind ego by VerifaiRange(1,4) diff --git a/tests/test_features.py b/tests/test_features.py index f47b3ed..9b02476 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -22,25 +22,25 @@ def test_fs_flatten_fixed_dimension(): 'a': Feature(DiscreteBox([0, 12])), 'b': Feature(Box((0, 1)), lengthDomain=DiscreteBox((0, 2))) }) - assert space.fixedFlattenedDimension == 5 + assert space.fixedFlattenedDimension == 4 sampler = FeatureSampler.randomSamplerFor(space) for i in range(100): point = sampler.getSample() flat = space.flatten(point, fixedDimension=True) assert type(flat) is tuple - assert len(flat) == 5 + assert len(flat) == 4 bLen = len(point.staticSample.b) - assert eval(space.meaningOfFlatCoordinate(0, pointName='point')) == point.staticSample.a[0] + assert eval(space.meaningOfFlatCoordinate(0)) == point.staticSample.a[0] assert flat[1] == bLen - assert eval(space.meaningOfFlatCoordinate(1, pointName='point')) == bLen + assert eval(space.meaningOfFlatCoordinate(1)) == bLen if bLen < 1: assert flat[2] is None else: - assert eval(space.meaningOfFlatCoordinate(2, pointName='point')) == point.staticSample.b[0][0] + assert eval(space.meaningOfFlatCoordinate(2)) == point.staticSample.b[0][0] if bLen < 2: assert flat[3] is None else: - assert eval(space.meaningOfFlatCoordinate(3, pointName='point')) == point.staticSample.b[1][0] + assert eval(space.meaningOfFlatCoordinate(3)) == point.staticSample.b[1][0] unflat = space.unflatten(flat, fixedDimension=True) assert point.staticSample == unflat.staticSample assert space.pandasIndexForFlatCoordinate(0) == ('a', 0) @@ -55,26 +55,26 @@ def test_fs_flatten_fixed_dimension2(): 'b': Feature(cat, lengthDomain=DiscreteBox((1, 3))), 'c': Feature(Box([-1, 1], [-3, 3])) }) - assert space.fixedFlattenedDimension == 7 + assert space.fixedFlattenedDimension == 6 sampler = FeatureSampler.randomSamplerFor(space) for i in range(100): point = sampler.getSample() flat = space.flatten(point, fixedDimension=True) assert type(flat) is tuple - assert len(flat) == 7 + assert len(flat) == 6 bLen = len(point.staticSample.b) assert 1 <= bLen <= 3 assert flat[0] == bLen - assert eval(space.meaningOfFlatCoordinate(0, pointName='point')) == bLen - assert eval(space.meaningOfFlatCoordinate(1, pointName='point')) == point.staticSample.b[0] + assert eval(space.meaningOfFlatCoordinate(0)) == bLen + assert eval(space.meaningOfFlatCoordinate(1)) == point.staticSample.b[0] if bLen < 2: assert flat[2] is None else: - assert eval(space.meaningOfFlatCoordinate(2, pointName='point')) == point.staticSample.b[1] + assert eval(space.meaningOfFlatCoordinate(2)) == point.staticSample.b[1] if bLen < 3: assert flat[3] is None else: - assert eval(space.meaningOfFlatCoordinate(3, pointName='point')) == point.staticSample.b[2] + assert eval(space.meaningOfFlatCoordinate(3)) == point.staticSample.b[2] unflat = space.unflatten(flat, fixedDimension=True) assert point.staticSample == unflat.staticSample assert space.pandasIndexForFlatCoordinate(0) == ('b', 'length') @@ -87,6 +87,47 @@ def test_fs_flatten_fixed_dimension2(): assert not any(space.coordinateIsNumerical(i) for i in range(1, 4)) assert all(space.coordinateIsNumerical(i) for i in range(4, 6)) +def test_fs_flatten_fixed_dimension_dynamic(): + space = FeatureSpace({ + 'a': Feature(DiscreteBox([0, 12])), + 'b': TimeSeriesFeature(Box((0, 1)), lengthDomain=DiscreteBox((0, 2))) + }, + timeBound=5 + ) + assert space.fixedFlattenedDimension == 13 + sampler = FeatureSampler.randomSamplerFor(space) + for i in range(100): + point = sampler.getSample() + duration = random.randint(0, 5) + for _ in range(duration): + point.getDynamicSample() + flat = space.flatten(point, fixedDimension=True) + assert type(flat) is tuple + assert len(flat) == 13 + assert eval(space.meaningOfFlatCoordinate(0)) == point.staticSample.a[0] + assert flat[1] == duration + for t in range(duration): + offset = 2*t + bLen = len(point.dynamicSampleHistory[t].b) + assert flat[2] == bLen + assert eval(space.meaningOfFlatCoordinate(2)) == bLen + if bLen < 1: + assert flat[offset+3] is None + else: + assert eval(space.meaningOfFlatCoordinate(offset+3)) == point.dynamicSampleHistory[t].b[0][0] + if bLen < 2: + assert flat[offset+4] is None + else: + assert eval(space.meaningOfFlatCoordinate(offset+4)) == point.dynamicSampleHistory[t].b[1][0] + unflat = space.unflatten(flat, fixedDimension=True) + assert point.staticSample == unflat.staticSample + assert all(point.dynamicSampleHistory[t] == unflat.dynamicSampleHistory[t] for t in range(duration)) + assert space.pandasIndexForFlatCoordinate(0) == ('a', 0) + assert space.pandasIndexForFlatCoordinate(2) == ('b', 'length') + assert space.pandasIndexForFlatCoordinate(3) == ('b', 0, 0) + assert space.pandasIndexForFlatCoordinate(4) == ('b', 1, 0) + assert all(space.coordinateIsNumerical(i) for i in range(4)) + def test_fs_distance(): box = Box([0, 10]) space = FeatureSpace({ 'a': Feature(box), 'b': Feature(box) }) diff --git a/tests/test_samplers.py b/tests/test_samplers.py index ec8c56b..91110eb 100644 --- a/tests/test_samplers.py +++ b/tests/test_samplers.py @@ -19,29 +19,18 @@ def test_feature_sampling(): sample = sampler.getSample() static_point = sample.staticSample - static_dict = static_point._asdict() - assert "a" in static_dict - assert "b" in static_dict - assert "c" not in static_dict - assert "d" not in static_dict - - assert len(static_dict["a"]) == 1 - assert 0 <= static_dict["a"][0] <= 12 - assert 0 <= len(static_dict["b"]) <= 2 - assert all(0 <= v[0] <= 1 for v in static_dict["b"]) + assert len(static_point.a) == 1 + assert 0 <= static_point.a[0] <= 12 + assert 0 <= len(static_point.b) <= 2 + assert all(0 <= v[0] <= 1 for v in static_point.b) for _ in range(space.timeBound): dynamic_point = sample.getDynamicSample() - dynamic_dict = dynamic_point._asdict() - assert "a" not in dynamic_dict - assert "b" not in dynamic_dict - assert "c" in dynamic_dict - assert "d" in dynamic_dict - assert len(dynamic_dict["c"]) == 1 - assert 2 <= dynamic_dict["c"][0] <= 5 - assert 0 <= len(dynamic_dict["d"]) <= 2 - assert all(5 <= v[0] <= 6 for v in dynamic_dict["d"]) + assert len(dynamic_point.c) == 1 + assert 2 <= dynamic_point.c[0] <= 5 + assert 0 <= len(dynamic_point.d) <= 2 + assert all(5 <= v[0] <= 6 for v in dynamic_point.d) ## Random sampling diff --git a/tests/utils.py b/tests/utils.py index 063814c..5a410aa 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -19,14 +19,10 @@ def checkSaveRestore(sampler, tmpdir, iterations=1): feedback = None for i in range(iterations): sampler.saveToFile(path) - # sample1 = sampler.nextSample(feedback) - # sample2 = sampler.nextSample(-1) sample1 = sampler.getSample() sample1.update(-1) sample2 = sampler.getSample() sampler = FeatureSampler.restoreFromFile(path) - # sample1b = sampler.nextSample(feedback) - # sample2b = sampler.nextSample(-1) sample1b = sampler.getSample() sample1b.update(-1) sample2b = sampler.getSample() @@ -35,11 +31,9 @@ def checkSaveRestore(sampler, tmpdir, iterations=1): assert sample1.staticSample == sample1b.staticSample assert sample2.staticSample == sample2b.staticSample sampler.saveToFile(path) - # sample3 = sampler.nextSample(1) sample3 = sampler.getSample() sampler = FeatureSampler.restoreFromFile(path) sample3b = sampler.getSample() assert sample3.staticSample not in (sample1.staticSample, sample2.staticSample) assert sample3.staticSample == sample3b.staticSample - # feedback = 1 sample3b.update(1) From 8e031614bc4e3141c8b23b2234f9a53dc51c5a84 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Wed, 24 Dec 2025 14:46:59 -0800 Subject: [PATCH 10/11] Cleaned up staticSample references in tests. --- src/verifai/features/features.py | 14 ++++++++------ tests/scenic/test_scenic.py | 16 ++++++++-------- tests/test_crossEntropy.py | 8 ++++---- tests/test_features.py | 18 +++++++++--------- tests/test_grid.py | 4 ++-- tests/test_samplers.py | 12 +++++++++--- tests/test_simulatedAnnealing.py | 2 +- 7 files changed, 41 insertions(+), 33 deletions(-) diff --git a/src/verifai/features/features.py b/src/verifai/features/features.py index ad2de4d..2b08b58 100644 --- a/src/verifai/features/features.py +++ b/src/verifai/features/features.py @@ -1037,7 +1037,7 @@ def update(self, rho): def __getattr__(self, attr): space = super().__getattribute__("space") if attr in space.staticFeatureNamed: - return getattr(space.staticSample, attr) + return getattr(self.staticSample, attr) elif attr in space.dynamicFeatureNamed: class DynamicFeatureHelper: def __init__(self, dynamicSampleHistory, attr): @@ -1045,6 +1045,8 @@ def __init__(self, dynamicSampleHistory, attr): self.attr = attr def __getitem__(self, i): + if i > len(self.dynamicSampleHistory): + raise ValueError("Attempting to access dynamic sample value that has not been sampled.") return getattr(self.dynamicSampleHistory[i], self.attr) return DynamicFeatureHelper(self.dynamicSampleHistory, attr) @@ -1269,19 +1271,19 @@ def meaningOfFlatCoordinate(self, index, pointName='point'): domain = feature.domain if feature.lengthDomain: if index == 0: - return f'len({pointName}.staticSample.{name})' + return f'len({pointName}.{name})' else: index -= 1 elem = index // domain.flattenedDimension if elem < feature.maxLength: subIndex = index % domain.flattenedDimension - subPoint = f'{pointName}.staticSample.{name}[{elem}]' + subPoint = f'{pointName}.{name}[{elem}]' return domain.meaningOfFlatCoordinate(subIndex, pointName=subPoint) index -= feature.maxLength * domain.flattenedDimension else: if index < domain.flattenedDimension: - subPoint = f'{pointName}.staticSample.{name}' + subPoint = f'{pointName}.{name}' return domain.meaningOfFlatCoordinate(index, pointName=subPoint) index -= domain.flattenedDimension @@ -1301,7 +1303,7 @@ def meaningOfFlatCoordinate(self, index, pointName='point'): elem = index // domain.flattenedDimension if elem < feature.maxLength: subIndex = index % domain.flattenedDimension - subPoint = f'{pointName}.dynamicSampleHistory[{time_i}].{name}[{elem}]' + subPoint = f'{pointName}.{name}[{time_i}][{elem}]' return domain.meaningOfFlatCoordinate(subIndex, pointName=subPoint) index -= feature.maxLength * domain.flattenedDimension @@ -1309,7 +1311,7 @@ def meaningOfFlatCoordinate(self, index, pointName='point'): index -= 1 for time_i in range(self.timeBound): if index < domain.flattenedDimension: - subPoint = f'{pointName}.dynamicSampleHistory[{time_i}].{name}' + subPoint = f'{pointName}.{name}[{time_i}]' return domain.meaningOfFlatCoordinate(index, pointName=subPoint) index -= domain.flattenedDimension diff --git a/tests/scenic/test_scenic.py b/tests/scenic/test_scenic.py index 951e184..c1fb3f7 100644 --- a/tests/scenic/test_scenic.py +++ b/tests/scenic/test_scenic.py @@ -16,7 +16,7 @@ def test_objects(new_Object): maxIterations=1 ) sample = sampler.getSample() - objects = sample.staticSample.objects + objects = sample.objects assert len(objects) == 1 pos = objects.object0.position assert type(pos) is tuple @@ -28,7 +28,7 @@ def test_params(new_Object): f'ego = {new_Object}', maxIterations=1 ) - sample = sampler.getSample().staticSample + sample = sampler.getSample() x = sample.params.x assert type(x) is float assert 3 <= x <= 5 @@ -39,7 +39,7 @@ def test_quoted_param(new_Object): f'ego = {new_Object}', maxIterations=1 ) - sample = sampler.getSample().staticSample + sample = sampler.getSample() v = sampler.paramDictForSample(sample)['x/y'] assert type(v) is float assert 3 <= v <= 5 @@ -49,7 +49,7 @@ def test_lists(new_Object): f'ego = {new_Object} with foo [1, -1, 3.3]', maxIterations=1 ) - sample = sampler.getSample().staticSample + sample = sampler.getSample() foo = sample.objects.object0.foo assert type(foo) is tuple assert foo == pytest.approx((1, -1, 3.3)) @@ -69,7 +69,7 @@ def test_object_order(new_Object): maxIterations=1 ) sample = sampler.getSample() - objects = sample.staticSample.objects + objects = sample.objects assert len(objects) == 11 for i in range(len(objects)): name = ScenicSampler.nameForObject(i) @@ -93,9 +93,9 @@ def test_active_sampling(new_Object): maxIterations=1 ) def f(sample): - return -1 if sample.staticSample.objects.object0.position[0] < 0 else 1 + return -1 if sample.objects.object0.position[0] < 0 else 1 samples = sampleWithFeedback(sampler, 120, f) - xs = [sample.staticSample.objects.object0.position[0] for sample in samples] + xs = [sample.objects.object0.position[0] for sample in samples] assert all(-1 <= x <= 1 for x in xs) assert any(x > 0 for x in xs) assert 66 <= sum(x < 0 for x in xs[50:]) @@ -112,7 +112,7 @@ def test_active_save_restore(new_Object, tmpdir): def runSampler(sampler): for i in range(3): - sample = sampler.getSample().staticSample + sample = sampler.getSample() print(f'Sample #{i}:') print(sample) diff --git a/tests/test_crossEntropy.py b/tests/test_crossEntropy.py index 926e8ff..6238b2e 100644 --- a/tests/test_crossEntropy.py +++ b/tests/test_crossEntropy.py @@ -17,8 +17,8 @@ def test_crossEntropy(): }) def f(sample): - print(sample.staticSample.cars[0].heading[0] - 0.75) - return abs(sample.staticSample.cars[0].heading[0] - 0.75) + print(sample.cars[0].heading[0] - 0.75) + return abs(sample.cars[0].heading[0] - 0.75) ce_params = DotMap() ce_params.alpha =0.9 @@ -37,8 +37,8 @@ def test_feedback_multiple_lengths(): }) def f(sample): - assert 1 <= len(sample.staticSample.a) <= 2 - return -1 if len(sample.staticSample.a) == 1 and sample.staticSample.a[0][0] < 0.5 else 1 + assert 1 <= len(sample.a) <= 2 + return -1 if len(sample.a) == 1 and sample.a[0][0] < 0.5 else 1 ce_params = DotMap(alpha=0.5, thres=0) ce_params.cont.buckets = 2 diff --git a/tests/test_features.py b/tests/test_features.py index 9b02476..1497d79 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -29,18 +29,18 @@ def test_fs_flatten_fixed_dimension(): flat = space.flatten(point, fixedDimension=True) assert type(flat) is tuple assert len(flat) == 4 - bLen = len(point.staticSample.b) - assert eval(space.meaningOfFlatCoordinate(0)) == point.staticSample.a[0] + bLen = len(point.b) + assert eval(space.meaningOfFlatCoordinate(0)) == point.a[0] assert flat[1] == bLen assert eval(space.meaningOfFlatCoordinate(1)) == bLen if bLen < 1: assert flat[2] is None else: - assert eval(space.meaningOfFlatCoordinate(2)) == point.staticSample.b[0][0] + assert eval(space.meaningOfFlatCoordinate(2)) == point.b[0][0] if bLen < 2: assert flat[3] is None else: - assert eval(space.meaningOfFlatCoordinate(3)) == point.staticSample.b[1][0] + assert eval(space.meaningOfFlatCoordinate(3)) == point.b[1][0] unflat = space.unflatten(flat, fixedDimension=True) assert point.staticSample == unflat.staticSample assert space.pandasIndexForFlatCoordinate(0) == ('a', 0) @@ -62,19 +62,19 @@ def test_fs_flatten_fixed_dimension2(): flat = space.flatten(point, fixedDimension=True) assert type(flat) is tuple assert len(flat) == 6 - bLen = len(point.staticSample.b) + bLen = len(point.b) assert 1 <= bLen <= 3 assert flat[0] == bLen assert eval(space.meaningOfFlatCoordinate(0)) == bLen - assert eval(space.meaningOfFlatCoordinate(1)) == point.staticSample.b[0] + assert eval(space.meaningOfFlatCoordinate(1)) == point.b[0] if bLen < 2: assert flat[2] is None else: - assert eval(space.meaningOfFlatCoordinate(2)) == point.staticSample.b[1] + assert eval(space.meaningOfFlatCoordinate(2)) == point.b[1] if bLen < 3: assert flat[3] is None else: - assert eval(space.meaningOfFlatCoordinate(3)) == point.staticSample.b[2] + assert eval(space.meaningOfFlatCoordinate(3)) == point.b[2] unflat = space.unflatten(flat, fixedDimension=True) assert point.staticSample == unflat.staticSample assert space.pandasIndexForFlatCoordinate(0) == ('b', 'length') @@ -104,7 +104,7 @@ def test_fs_flatten_fixed_dimension_dynamic(): flat = space.flatten(point, fixedDimension=True) assert type(flat) is tuple assert len(flat) == 13 - assert eval(space.meaningOfFlatCoordinate(0)) == point.staticSample.a[0] + assert eval(space.meaningOfFlatCoordinate(0)) == point.a[0] assert flat[1] == duration for t in range(duration): offset = 2*t diff --git a/tests/test_grid.py b/tests/test_grid.py index 0378a9e..007b292 100644 --- a/tests/test_grid.py +++ b/tests/test_grid.py @@ -16,7 +16,7 @@ def test_grid(): try: sample = sampler.getSample() sample.update(None) - sample = sample.staticSample + sample = sample dict_samples[(sample.weather[0], sample.car_positions[0], sample.car_positions[1])] = 0 except TerminationException: @@ -50,7 +50,7 @@ def f(sample): for i in range(21): sample = sampler.getSample() sample.update(None) - sample = sample.staticSample + sample = sample samples.append(sample) y_samples.append(f(sample)) diff --git a/tests/test_samplers.py b/tests/test_samplers.py index 91110eb..08b6c40 100644 --- a/tests/test_samplers.py +++ b/tests/test_samplers.py @@ -17,7 +17,7 @@ def test_feature_sampling(): sampler = FeatureSampler.randomSamplerFor(space) sample = sampler.getSample() - static_point = sample.staticSample + static_point = sample assert len(static_point.a) == 1 assert 0 <= static_point.a[0] <= 12 @@ -32,6 +32,12 @@ def test_feature_sampling(): assert 0 <= len(dynamic_point.d) <= 2 assert all(5 <= v[0] <= 6 for v in dynamic_point.d) + for i in range(space.timeBound): + assert len(sample.c[i]) == 1 + assert 2 <= sample.c[i][0] <= 5 + assert 0 <= len(sample.d[i]) <= 2 + assert all(5 <= v[0] <= 6 for v in sample.d[i]) + ## Random sampling def test_domain_random(): @@ -97,8 +103,8 @@ def check(samples): assert any(len(sample.b) == 1 for sample in samples) assert any(len(sample.b) == 2 for sample in samples) - check([sampler.getSample().staticSample for i in range(100)]) - check(list(s.staticSample for s in itertools.islice(sampler, 100))) + check([sampler.getSample() for i in range(100)]) + check(list(s for s in itertools.islice(sampler, 100))) def test_random_restore(tmpdir): space = FeatureSpace({ diff --git a/tests/test_simulatedAnnealing.py b/tests/test_simulatedAnnealing.py index 562d09d..61ade37 100644 --- a/tests/test_simulatedAnnealing.py +++ b/tests/test_simulatedAnnealing.py @@ -45,7 +45,7 @@ def test_simulatedAnnealing(): }) def f(sample): - return sample.staticSample.cars[0].heading[0] - 0.75 + return sample.cars[0].heading[0] - 0.75 sampler = FeatureSampler.simulatedAnnealingSamplerFor(space, sa_params) From 26e63d45631050a18cde56ba8e6262c7ca136037 Mon Sep 17 00:00:00 2001 From: Eric Vin Date: Wed, 24 Dec 2025 14:47:50 -0800 Subject: [PATCH 11/11] Additional cleanup --- tests/test_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_grid.py b/tests/test_grid.py index 007b292..3477efc 100644 --- a/tests/test_grid.py +++ b/tests/test_grid.py @@ -63,6 +63,6 @@ def test_grid_non_standardizable(): 'b': Feature(FilteredDomain(Box([0,1]), lambda x: x[0] > 0.5)) }) sampler = FeatureSampler.gridSamplerFor(space) - samples = [s.staticSample for s in sampler] + samples = [s for s in sampler] assert len(samples) == 13 assert all(sample.b[0] > 0.5 for sample in samples)