55  setup_kwargs = dict(zip_safe=0)
 56else:
 57  from distutils.core import setup
 58  setup_kwargs = dict() 59
 60
 61storage_dirs = [ ('storage/ceres/dummy.txt', []), ('storage/whisper/dummy.txt',[]),

 54  from setuptools import setup
 55  setup_kwargs = dict(zip_safe=0)
 56else:
 57  from distutils.core import setup 58  setup_kwargs = dict()
 59
 60

 52
 53if os.environ.get('USE_SETUPTOOLS'):
 54  from setuptools import setup
 55  setup_kwargs = dict(zip_safe=0) 56else:
 57  from distutils.core import setup
 58  setup_kwargs = dict()

 51
 52
 53if os.environ.get('USE_SETUPTOOLS'):
 54  from setuptools import setup 55  setup_kwargs = dict(zip_safe=0)
 56else:
 57  from distutils.core import setup

229  """Same as AggregatedConsistentHashingRouter but using FastHashRing."""
230  plugin_name = 'fast-aggregated-hashing'
231
232  def __init__(self, settings):233    super(FastAggregatedHashingRouter, self).__init__(settings)
234    self.hash_router.ring = FastHashRing(settings)

227
228class FastAggregatedHashingRouter(AggregatedConsistentHashingRouter):
229  """Same as AggregatedConsistentHashingRouter but using FastHashRing."""
230  plugin_name = 'fast-aggregated-hashing'231
232  def __init__(self, settings):
233    super(FastAggregatedHashingRouter, self).__init__(settings)

226
227
228class FastAggregatedHashingRouter(AggregatedConsistentHashingRouter):
229  """Same as AggregatedConsistentHashingRouter but using FastHashRing."""230  plugin_name = 'fast-aggregated-hashing'
231
232  def __init__(self, settings):

220  """Same as ConsistentHashingRouter but using FastHashRing."""
221  plugin_name = 'fast-hashing'
222
223  def __init__(self, settings):224    super(FastHashingRouter, self).__init__(settings)
225    self.ring = FastHashRing(settings)
226

218
219class FastHashingRouter(ConsistentHashingRouter):
220  """Same as ConsistentHashingRouter but using FastHashRing."""
221  plugin_name = 'fast-hashing'222
223  def __init__(self, settings):
224    super(FastHashingRouter, self).__init__(settings)

217
218
219class FastHashingRouter(ConsistentHashingRouter):
220  """Same as ConsistentHashingRouter but using FastHashRing."""221  plugin_name = 'fast-hashing'
222
223  def __init__(self, settings):

213    seed = self._hash(key) % len(self.nodes)
214
215    for n in xrange(seed, seed + len(self.nodes)):
216      yield self.sorted_nodes[n % len(self.sorted_nodes)][1]217
218
219class FastHashingRouter(ConsistentHashingRouter):

208
209  def get_nodes(self, key):
210    if not self.nodes:
211      return212
213    seed = self._hash(key) % len(self.nodes)
214

206    self.nodes.discard(node)
207    self._update_nodes()
208
209  def get_nodes(self, key):210    if not self.nodes:
211      return
212

202    self.nodes.add(node)
203    self._update_nodes()
204
205  def remove_node(self, node):206    self.nodes.discard(node)
207    self._update_nodes()
208

198      key=lambda v: v[0]
199    )
200
201  def add_node(self, node):202    self.nodes.add(node)
203    self._update_nodes()
204

192  def _hash(self, key):
193    return carbonHash(key, self.hash_type)
194
195  def _update_nodes(self):196    self.sorted_nodes = sorted(
197      [(self._hash(str(n)), n) for n in self.nodes],
198      key=lambda v: v[0]

189    self.sorted_nodes = []
190    self.hash_type = settings.ROUTER_HASH_TYPE or 'mmh3_ch'
191
192  def _hash(self, key):193    return carbonHash(key, self.hash_type)
194
195  def _update_nodes(self):

184  and not the number of metrics to rebalance.
185  """
186
187  def __init__(self, settings):188    self.nodes = set()
189    self.sorted_nodes = []
190    self.hash_type = settings.ROUTER_HASH_TYPE or 'mmh3_ch'

175
176
177class FastHashRing(object):
178  """A very fast hash 'ring'.179
180  Instead of trying to avoid rebalancing data when changing
181  the list of nodes we try to making routing as fast as we

171        destinations.add(destination)
172
173    for destination in destinations:
174      yield destination175
176
177class FastHashRing(object):

167    # get consistent hashing destinations based on aggregate forms
168    destinations = set()
169    for resolved_metric in resolved_metrics:
170      for destination in self.hash_router.getDestinations(resolved_metric):171        destinations.add(destination)
172
173    for destination in destinations:

162    # if the metric will not be aggregated, send it raw
163    # (will pass through aggregation)
164    if len(resolved_metrics) == 0:
165      resolved_metrics.append(key)166
167    # get consistent hashing destinations based on aggregate forms
168    destinations = set()

156      aggregate_metric = rule.get_aggregate_metric(key)
157      if aggregate_metric is None:
158        continue
159      else:160        resolved_metrics.append(aggregate_metric)
161
162    # if the metric will not be aggregated, send it raw

154    resolved_metrics = []
155    for rule in self.agg_rules_manager.rules:
156      aggregate_metric = rule.get_aggregate_metric(key)
157      if aggregate_metric is None:158        continue
159      else:
160        resolved_metrics.append(aggregate_metric)

153    # resolve metric to aggregate forms
154    resolved_metrics = []
155    for rule in self.agg_rules_manager.rules:
156      aggregate_metric = rule.get_aggregate_metric(key)157      if aggregate_metric is None:
158        continue
159      else: