Coverage for gwcelery/conf/__init_

1"""GWCelery application configuration.

3This module defines configuration variables and default values, including both

4:doc:`generic options for Celery <celery:userguide/configuration>` as well as

5options that control the behavior of specific GWCelery :mod:`~gwcelery.tasks`.

7To override the configuration, define the ``CELERY_CONFIG_MODULE`` environment

8variable to the fully qualified name of any Python module that can be located

9in :obj:`sys.path`, including any of the following presets:

11 * :mod:`gwcelery.conf.playground`

12 * :mod:`gwcelery.conf.production`

13 * :mod:`gwcelery.conf.test`

14 * :mod:`gwcelery.conf.dev` (the default)

15"""

17import getpass

18import os

20from igwn_alert.client import DEFAULT_SERVER as DEFAULT_IGWN_ALERT_SERVER

22# Celery application settings.

24# Task tombstones expire after 2 hours.

25# Celery's default setting of 1 day could cause the Redis database to grow too

26# large because we pass large byte strings as task arguments and return values.

27result_expires = 7200

29# Use pickle serializer, because it supports byte values.

30accept_content = ['json', 'pickle']

31event_serializer = 'json'

32result_serializer = 'pickle'

33task_serializer = 'pickle'

35# Compress tasks to reduce bandwidth in and out of Redis.

36result_compression = task_compression = 'zstandard'

38# Task priority settings.

39task_inherit_parent_priority = True

40task_default_priority = 0

41task_queue_max_priority = 1

42broker_transport_options = {

43 'priority_steps': list(range(task_queue_max_priority + 1))

44}

46worker_proc_alive_timeout = 8

47"""The timeout when waiting for a new worker process to start up."""

49worker_log_format = "[%(asctime)s: %(levelname)s/%(processName)s/%(threadName)s] %(message)s" # noqa: E501

50"""Custom worker log format that includes the thread name."""

52# GWCelery-specific settings.

54condor_accounting_group = 'ligo.dev.o4.cbc.pe.bayestar'

55"""HTCondor accounting group for Celery workers launched with condor_submit."""

57expose_to_public = False

58"""Set to True if events meeting the public alert threshold really should be

59exposed to the public."""

61igwn_alert_server = DEFAULT_IGWN_ALERT_SERVER

62"""IGWN alert server"""

64igwn_alert_noauth = False

65"""IGWN alert server no-authetication"""

67igwn_alert_group = 'gracedb-test01'

68"""IGWN alert group."""

70gracedb_host = 'gracedb-test01.igwn.org'

71"""GraceDB host."""

73gwskynet_upper_far_threshold = 1 / 21600

74"""GWSkyNet will only annotate events with FARs below this value."""

76gwskynet_snr_threshold = 4.5

77"""GWSkyNet will only annotate events with individual SNRs above this value."""

79gwskynet_network_snr_threshold = 7.0

80"""GWSkyNet will only annotate events with network SNRs above this value."""

82create_mattermost_channel = False

83"""Do not create Mattermost channel."""

85kafka_consumer_config = {

86 'fermi': {'url': 'kafka://kafka.test.gcn.nasa.gov/'

87 'fermi.gbm.targeted.private.igwn', 'suffix': 'json'},

88 'swift': {'url': 'kafka://kafka.gcn.nasa.gov/'

89 'gcn.notices.swift.bat.guano', 'suffix': 'json'}

90}

91"""Kafka consumer configuration details. The keys describe the senders of the

92messages to be consumed. The values are a dictionary of the URL to listen to

93and information about the message serializer. NOTE: We will switch the Swift

94URL to a test topic once the topic is regularly written to."""

96views_manual_preferred_event_log_message = 'User {} queued a preferred event' \

97 ' change to {}.'

98"""Log message that is uploaded to GraceDB when a user manually changes the

99preferred event in the dashboard."""

100

101voevent_broadcaster_address = ':5342'

102"""The VOEvent broker will bind to this address to send GCNs.

103This should be a string of the form `host:port`. If `host` is empty,

104then listen on all available interfaces."""

105

106voevent_broadcaster_whitelist = []

107"""List of hosts from which the broker will accept connections.

108If empty, then completely disable the broker's broadcast capability."""

109

110voevent_receiver_address = '68.169.57.253:8099'

111"""The VOEvent listener will connect to this address to receive GCNs. For

112options, see `GCN's list of available VOEvent servers

113<https://gcn.gsfc.nasa.gov/voevent.html#tc2>`_. If this is an empty string,

114then completely disable the GCN listener."""

115

116email_host = 'imap.gmail.com'

117"""IMAP hostname to receive the GCN e-mail notice formats."""

118

119superevent_d_t_start = {'gstlal': 1.0,

120 'spiir': 1.0,

121 'pycbc': 1.0,

122 'mbta': 1.0}

123"""Pipeline based lower extent of superevent segments.

124For cwb and lib this is decided from extra attributes."""

125

126superevent_d_t_end = {'gstlal': 1.0,

127 'spiir': 1.0,

128 'pycbc': 1.0,

129 'mbta': 1.0}

130"""Pipeline based upper extent of superevent segments

131For cwb and lib this is decided from extra attributes."""

132

133superevent_query_d_t_start = 100.

134"""Lower extent of superevents query"""

135

136superevent_query_d_t_end = 100.

137"""Upper extent of superevents query"""

138

139superevent_default_d_t_start = 1.0

140"""Default lower extent of superevent segments"""

141

142superevent_default_d_t_end = 1.0

143"""Default upper extent for superevent segments"""

144

145superevent_far_threshold = 1 / 3600

146"""Maximum false alarm rate to consider events superevents."""

147

148superevent_candidate_preference = {

149 'cbc': {'allsky': 2, 'earlywarning': 1, 'ssm': 1, 'mdc': 1},

150 'burst': {'allsky': 0, 'bbh': 0, 'mdc': 0},

151}

152"""Group/search preference for individual candidates. This is

153used by :meth:`gwcelery.tasks.superevents.keyfunc` to sort

154candidates for the preferred event before a ranking statistic is used."""

155

156significant_alert_far_threshold = {

157 'cbc': {

158 'allsky': 1 / (30 * 86400),

159 'earlywarning': 1 / (30 * 86400),

160 'mdc': 1 / (30 * 86400),

161 'ssm': -1 * float('inf') # SSM public alerts disabled until approved

162 },

163 'burst': {

164 'allsky': 1 / (365 * 86400),

165 'bbh': 1 / (30 * 86400)

166 },

167 'test': {

168 'allsky': 1 / (30 * 86400),

169 'bbh': 1 / (30 * 86400),

170 'earlywarning': 1 / (30 * 86400),

171 'ssm': 1 / (30 * 86400)

172 }

173}

174"""Group and search specific maximum false alarm rate to consider sending

175significant alerts. A threshold of negative infinity disables alerts."""

176

177significant_alert_trials_factor = {

178 'cbc': {'allsky': 5,

179 'earlywarning': 3,

180 'mdc': 5,

181 'ssm': 2},

182 'burst': {'allsky': 2,

183 'bbh': 5}

184}

185"""Trials factor corresponding to trigger categories. The CBC AllSky and Burst

186BBH searches are treated as one group with a common trials factor. CBC AllSky

187pipelines are gstlal, pycbc, mbta, and raven. The Burst BBH pipeline is cwb.

188CBC EarlyWarning pipelines are gstlal, pycbc, and mbta. CBC SSM pipelines are

189gstlal and mbta. The Burst AllSky searches are treated as one group with one

190trials factor. The Burst AllSky piplines are cwb and raven."""

191

192preliminary_alert_trials_factor = {

193 'cbc': {'allsky': 5,

194 'earlywarning': 3,

195 'mdc': 5,

196 'ssm': 2},

197 'burst': {'allsky': 5,

198 'bbh': 5}

199}

200"""Trials factor for less significant alert categories. The CBC AllSky, Burst

201AllSky, and Burst BBH searches are all treated as one group with a shared

202trials factor. CBC AllSky pipelines are gstlal, pycbc, and mbta. Burst AllSky

203pipeline is cwb. The Burst BBH pipeline is cwb."""

204

205preliminary_alert_far_threshold = {

206 'cbc': {

207 'allsky': 2 / (1 * 86400) * preliminary_alert_trials_factor['cbc']['allsky'], # noqa: E501

208 'earlywarning': -1 * float('inf'),

209 'mdc': -1 * float('inf'),

210 'ssm': -1 * float('inf')

211 },

212 'burst': {

213 'allsky': 2 / (1 * 86400) * preliminary_alert_trials_factor

214 ['burst']['allsky'],

215 'bbh': 2 / (1 * 86400) * preliminary_alert_trials_factor

216 ['burst']['bbh']

217 },

218 'test': {

219 'allsky': 2 / (1 * 86400) * preliminary_alert_trials_factor

220 ['cbc']['allsky'],

221 'earlywarning': -1 * float('inf'),

222 'ssm': -1 * float('inf')

223 }

224}

225"""Group and search specific maximum false alarm rate to consider sending less

226significant alerts. Trials factors are included here to ensure events are sent

227with the false alarm rate initially listed and removing trials factors are from

228the threshold calculation. A threshold of negative infinity disables alerts."""

229

230snews_gw_far_threshold = 1 / (3600 * 24)

231"""Maximum false alarm rate for a superevent to send out a coincidence alert

232between an external SNEWS alert and the superevent."""

233

234sog_paper_far_threshold = {'gw': 1 / (10 * 365 * 86400),

235 'joint': 1 / (10000 * 365 * 86400)}

236"""False alarm rate thresholds for producing a manuscript of speed of gravity

237measurement in low-latency."""

238

239superevent_clean_up_timeout = 270.

240"""The orchestrator will wait this many seconds from the time of the

241application of the GCN_PRELIM_SENT label to revise the preferred

242event out of the accumulated events."""

243

244pe_timeout = 345.0

245"""The orchestrator will wait this many seconds from the time of the

246creation of a new superevent to the time that parameter estimation begins, in

247case the preferred event is updated with high latency."""

248

249rapidpe_timeout = 30.0

250"""The orchestrator will wait this many seconds from the time of the

251creation of a new superevent to the time rapidpe parameter estimation begins,

252in case the preferred event is updated with high latency."""

253

254

255check_vector_prepost = {'gstlal': [2, 2],

256 'spiir': [2, 2],

257 'pycbc': [2, 2],

258 'MBTA': [2, 2],

259 'oLIB': [1.5, 1.5],

260 'LIB': [1.5, 1.5],

261 'CWB': [1.5, 1.5],

262 'MLy': [1.5, 1.5],

263 'HardwareInjection': [2, 2],

264 'Swift': [2, 2],

265 'Fermi': [2, 2],

266 'INTEGRAL': [2, 2],

267 'SNEWS': [10, 10]}

268"""Seconds before and after the superevent start and end times which the DQ

269vector check will include in its check. Pipeline dependent."""

270

271uses_gatedhoft = {'gstlal': True,

272 'spiir': True,

273 'pycbc': True,

274 'MBTA': True,

275 'oLIB': False,

276 'LIB': False,

277 'CWB': True,

278 'MLy': False,

279 'HardwareInjection': False,

280 'Swift': False,

281 'Fermi': False,

282 'INTEGRAL': False,

283 'SNEWS': False}

284"""Whether or not a pipeline uses gated h(t). Determines whether or not

285the DMT-DQ_VECTOR will be analyzed for data quality."""

286

287llhoft_glob = '/dev/shm/kafka/{detector}_O3ReplayMDC/*.gwf'

288"""File glob for playground low-latency h(t) frames. Currently points

289to O3 MDC Mock Data Challange data.

290See https://git.ligo.org/emfollow/mock-data-challenge"""

291

292llhoft_channels = {

293 'H1:DMT-DQ_VECTOR': 'dmt_dq_vector_bits',

294 'L1:DMT-DQ_VECTOR': 'dmt_dq_vector_bits',

295 'H1:GDS-CALIB_STATE_VECTOR': 'ligo_state_vector_bits',

296 'L1:GDS-CALIB_STATE_VECTOR': 'ligo_state_vector_bits',

297 'V1:DQ_ANALYSIS_STATE_VECTOR': 'virgo_state_vector_bits'}

298"""Low-latency h(t) state vector configuration. This is a dictionary consisting

299of a channel and its bitmask, as defined in :mod:`gwcelery.tasks.detchar`."""

300

301idq_ok_channels = ['H1:IDQ-OK_OVL_10_2048',

302 'L1:IDQ-OK_OVL_10_2048']

303"""Low-latency iDQ OK channel names for O4. High bit indicates iDQ is ok."""

304

305idq_channels = ['H1:IDQ-FAP_OVL_10_2048',

306 'L1:IDQ-FAP_OVL_10_2048']

307"""Low-latency iDQ false alarm probability channel names for O4."""

308

309idq_fap_thresh = 0.01

310"""If FAP is below this threshold, and

311:obj:`~gwcelery.conf.idq_veto` for the pipeline is true, DQV will be labeled

312for the event.

313"""

314

315idq_veto = {'gstlal': False,

316 'spiir': False,

317 'pycbc': False,

318 'MBTA': False,

319 'oLIB': False,

320 'LIB': False,

321 'CWB': False,

322 'MLy': False,

323 'HardwareInjection': False,

324 'Swift': False,

325 'Fermi': False,

326 'INTEGRAL': False,

327 'SNEWS': False}

328"""If true for a pipeline, iDQ values below the threshold defined in

329:obj:`~gwcelery.conf.idq_fap_thresh` will cause DQV to be labeled.

330Currently all False, pending iDQ review (should be done before O3).

331"""

332

333low_latency_frame_types = {'H1': 'H1_O3ReplayMDC_llhoft',

334 'L1': 'L1_O3ReplayMDC_llhoft',

335 'V1': 'V1_O3ReplayMDC_llhoft'}

336"""Types of low latency frames used in Parameter Estimation (see

337:mod:`gwcelery.tasks.inference`) and in cache creation for detchar

338checks (see :mod:`gwcelery.tasks.detchar`).

339"""

340

341high_latency_frame_types = {'H1': None,

342 'L1': None,

343 'V1': None}

344"""Types of high latency frames used in Parameter Estimation and in cache

345creation for detchar checks. They do not exist for O3Replay data. (see

346:mod:`gwcelery.tasks.inference` and :mod:`gwcelery.tasks.detchar`)

347"""

348

349strain_channel_names = {'H1': 'H1:GDS-CALIB_STRAIN_INJ1_O3Replay',

350 'L1': 'L1:GDS-CALIB_STRAIN_INJ1_O3Replay',

351 'V1': 'V1:Hrec_hoft_16384Hz_INJ1_O3Replay'}

352"""Names of h(t) channels used in Parameter Estimation (see

353:mod:`gwcelery.tasks.inference`) and in detchar omegascan creation

354(see :mod:`gwcelery.tasks.detchar`)."""

355

356state_vector_channel_names = {'H1': 'H1:GDS-CALIB_STATE_VECTOR',

357 'L1': 'L1:GDS-CALIB_STATE_VECTOR',

358 'V1': 'V1:DQ_ANALYSIS_STATE_VECTOR'}

359"""Names of state vector channels used in Parameter Estimation (see

360:mod:`gwcelery.tasks.inference`)"""

361

362detchar_bit_definitions = {

363 'dmt_dq_vector_bits': {

364 'channel': 'DMT-DQ_VECTOR',

365 'bits': {

366 1: 'NO_OMC_DCPD_ADC_OVERFLOW',

367 2: 'NO_DMT-ETMY_ESD_DAC_OVERFLOW'

368 },

369 'description': {

370 'NO_OMC_DCPD_ADC_OVERFLOW': 'OMC DCPC ADC not overflowing',

371 'NO_DMT-ETMY_ESD_DAC_OVERFLOW': 'ETMY ESD DAC not overflowing'

372 }

373 },

374 'ligo_state_vector_bits': {

375 'channel': 'GDS-CALIB_STATE_VECTOR',

376 'bits': {

377 0: 'HOFT_OK',

378 1: 'OBSERVATION_INTENT',

379 5: 'NO_STOCH_HW_INJ',

380 6: 'NO_CBC_HW_INJ',

381 7: 'NO_BURST_HW_INJ',

382 8: 'NO_DETCHAR_HW_INJ'

383 },

384 'description': {

385 'HOFT_OK': 'h(t) was successfully computed',

386 'OBSERVATION_INTENT': '"observation intent" button is pushed',

387 'NO_STOCH_HW_INJ': 'No stochastic HW injection',

388 'NO_CBC_HW_INJ': 'No CBC HW injection',

389 'NO_BURST_HW_INJ': 'No burst HW injection',

390 'NO_DETCHAR_HW_INJ': 'No HW injections for detector characterization' # noqa: E501

391 }

392 },

393 'virgo_state_vector_bits': {

394 'channel': 'DQ_ANALYSIS_STATE_VECTOR',

395 'bits': {

396 0: 'HOFT_OK',

397 1: 'OBSERVATION_INTENT',

398 5: 'NO_STOCH_HW_INJ',

399 6: 'NO_CBC_HW_INJ',

400 7: 'NO_BURST_HW_INJ',

401 8: 'NO_DETCHAR_HW_INJ',

402 10: 'GOOD_DATA_QUALITY_CAT1'

403 },

404 'description': {

405 'HOFT_OK': 'h(t) was successfully computed',

406 'OBSERVATION_INTENT': '"observation intent" button is pushed',

407 'NO_STOCH_HW_INJ': 'No stochastic HW injection',

408 'NO_CBC_HW_INJ': 'No CBC HW injection',

409 'NO_BURST_HW_INJ': 'No burst HW injection',

410 'NO_DETCHAR_HW_INJ': 'No HW injections for detector characterization', # noqa: E501

411 'GOOD_DATA_QUALITY_CAT1': 'Good data quality (CAT1 type)'

412 }

413 }

414}

415"""Bit definitions for detchar checks"""

416

417omegascan_durations = [(0.75, 0.25), (1.5, 0.5), (7.5, 2.5)]

418"""Durations for omegascans, before and after t0"""

419

420pe_results_path = os.path.join(os.getenv('HOME'), 'public_html/online_pe')

421"""Path to the results of Parameter Estimation (see

422:mod:`gwcelery.tasks.inference`)"""

423

424pe_results_url = ('https://ldas-jobs.ligo.caltech.edu/~{}/'

425 'online_pe/').format(getpass.getuser())

426"""URL of page where all the results of Parameter Estimation are outputted

427(see :mod:`gwcelery.tasks.inference`)"""

428

429raven_coincidence_windows = {'GRB_CBC': [-5, 1],

430 'GRB_CBC_SubFermi': [-11, 1],

431 'GRB_CBC_SubSwift': [-20, 10],

432 'GRB_Burst': [-600, 60],

433 'SNEWS': [-10, 10]}

434"""Time coincidence windows passed to ligo-raven. External events and

435superevents of the appropriate type are considered to be coincident if

436within time window of each other."""

437

438raven_ext_rates = {

439 'GRB': 305 / (3600 * 24 * 365), # 305 / yr

440 'MDC': 305 / (3600 * 24 * 365), # 305 / yr

441 'SubGRB': 370 / (3600 * 24 * 365) # 370 / yr

442}

443"""Expected rates of astrophysical external triggers submitted for each search.

444These potentially include multiple satellites and experiments (e.g. for GRBs,

445Fermi+Swift+INTEGRAL), accounting for multiple detections of the same

446astrophysical event. For more details on methodology, see:

447https://dcc.ligo.org/T1900297."""

448

449raven_targeted_far_thresholds = {

450 'GW': {

451 'Fermi': preliminary_alert_far_threshold['cbc']['allsky'],

452 'Swift': preliminary_alert_far_threshold['cbc']['allsky']

453 },

454 'GRB': {

455 'Fermi': 1 / 10000,

456 'Swift': 1 / 1000

457 }

458}

459"""Max FAR thresholds used for the subthreshold targeted searches with Fermi

460and Swift. Since we only listen to CBC low significance alerts, we use that

461FAR threshold for now. Note that Swift current listens to events with the

462threshold before and Fermi after trials factors."""

463

464mock_events_simulate_multiple_uploads = False

465"""If True, then upload each mock event several times in rapid succession with

466random jitter in order to simulate multiple pipeline uploads."""

467

468only_alert_for_mdc = False

469"""If True, then only sends alerts for MDC events. Useful for times outside

470of observing runs."""

471

472joint_mdc_freq = 2

473"""Determines how often an external MDC event will be created near an

474MDC superevent to test the RAVEN alert pipeline, i.e for every x

475MDC superevents an external MDC event is created."""

476

477joint_O3_replay_freq = 10

478"""Determines how often an external replay event will be created near an

479superevent to test the RAVEN alert pipeline, i.e for every x

480O3 replay superevents an external MDC event is created."""

481

482condor_retry_kwargs = dict(

483 max_retries=80, retry_backoff=True, retry_jitter=False,

484 retry_backoff_max=600

485)

486"""Retry settings of condor.submit task. With these settings, a condor job is

487no longer tracked ~12 hours after it starts."""

488

489rapidpe_condor_retry_kwargs = dict(

490 max_retries=125, retry_backoff=True, retry_jitter=False,

491 retry_backoff_max=30

492)

493"""Retry settings of condor.submit_rapidpe task. With these settings, a

494condor job is no longer tracked ~1 hours after it starts. This is used

495for RapidPE."""

496

497rapidpe_settings = {

498 'run_mode': 'online',

499 'accounting_group': 'ligo.dev.o4.cbc.pe.lalinferencerapid',

500 'use_cprofile': True,

501}

502"""Config settings used for rapidpe"""

503

504# Delete imported modules so that they do not pollute the config object

505del os, getpass

Coverage for gwcelery/conf/init.py: 100%

75 statements