star-hitran

commit 5f0fd2881adc5b4c55c4d9afa47709bcb1a60dc2
parent 08240bde359b290d3a8a790d23a2e443b305cda6
Author: Vincent Forest <vincent.forest@meso-star.com>
Date:   Wed,  4 Mar 2026 15:06:46 +0100

Update quantification of self-broadening half width

It was assumed that its actual values were within [0,1[. However, H2O
lines in the HITRAN database are outside this domain. Thus, certain bits
must be used to encode the entire part. In fact 3 bits must be reserved
since a value greater than four has been observed.

And since there is no reason for this quantification to work in all
situations, dynamically check whether the quantification of lines does
not present any problem, and otherwise reject problematic lines with a
warning message. The caller will then be able to alert the need to
rethink the quantification without being blocked by an unexpected
dataset.

This seems a good tradeoff since with the previous quantification, only
256 lines would have been rejected, out of more than 400 thousand.

Diffstat:
M
src/shtr_line_list.c
 | 
59
+++++++++++++++++++++++++++++++++++++++++++++--------------

1 file changed, 45 insertions(+), 14 deletions(-)
diff --git a/src/shtr_line_list.c b/src/shtr_line_list.c
@@ -174,10 +174,15 @@ get_line(const struct shtr_line_list* list, const size_t i)
   return (struct line*)(darray_charp_cdata_get(&list->blocks))[iblock] + iline;
 }
 
-static void
-line_encode(const struct shtr_line* line, struct line* line_encoded)
+static res_T
+line_encode
+  (const struct shtr* shtr, /* For logging */
+   const struct txtrdr* txtrdr, /* For logging */
+   const struct shtr_line* line,
+   struct line* line_encoded)
 {
   union { float flt; int32_t i32; } ucast;
+  res_T res = RES_OK;
   ASSERT(line && line_encoded);
 
   /* Keep the wavenumber and the intensity as it */
@@ -190,15 +195,29 @@ line_encode(const struct shtr_line* line, struct line* line_encoded)
 
   /* Store gamma_air as an unsigned fixed-point number (0:14), i.e., 0 bit for
    * the integer part and 14 bits for the fractional part */
-  ASSERT((int64_t)line->gamma_air == 0 && line->gamma_air >= 0);
-  line_encoded->gair14_gself14_isoid4 =
-    ((int32_t)float2fix(line->gamma_air, 0, 14) & (BIT_I32(14)-1)) << 18;
+  if((int64_t)line->gamma_air != 0) {
+    WARN(shtr,
+      "%s:%lu: invalid gamma air %g: encoding expects a value in [0,1[\n",
+      txtrdr_get_name(txtrdr), txtrdr_get_line_num(txtrdr), line->gamma_air);
+    res = RES_BAD_OP;
+  } else {
+    ASSERT(line->gamma_air >= 0);
+    line_encoded->gair14_gself14_isoid4 =
+      ((int32_t)float2fix(line->gamma_air, 0, 14) & (BIT_I32(14)-1)) << 18;
+  }
 
-  /* Store gamma_self as an unsigned fixed-point number (0:14), i.e., 0 bit for
-   * the integer part and 14 bits for the fractional part */
-  ASSERT((int64_t)line->gamma_self == 0 && line->gamma_self >= 0);
-  line_encoded->gair14_gself14_isoid4 |=
-    ((int32_t)float2fix(line->gamma_self, 0, 14) & (BIT_I32(14)-1)) << 4;
+  /* Store gamma_self as an unsigned fixed-point number (3:11), i.e., 3 bit for
+   * the integer part and 11 bits for the fractional part */
+  if((int64_t)line->gamma_self > 8) {
+    WARN(shtr,
+      "%s:%lu: invalid gamma self %g: encoding expects a value in [0,8[\n",
+      txtrdr_get_name(txtrdr), txtrdr_get_line_num(txtrdr), line->gamma_self);
+    res = RES_BAD_OP;
+  } else {
+    ASSERT(line->gamma_self >= 0);
+    line_encoded->gair14_gself14_isoid4 |=
+      ((int32_t)float2fix(line->gamma_self, 3, 11) & (BIT_I32(14)-1)) << 4;
+  }
 
   /* Store the isotope id on 4 bits */
   ASSERT(line->isotope_id_local < 16);
@@ -213,6 +232,8 @@ line_encode(const struct shtr_line* line, struct line* line_encoded)
   ASSERT(line->molecule_id < 128);
   ucast.i32 |= (int32_t)line->molecule_id & (BIT_I32(7)-1);
   line_encoded->nair25_molid7 = ucast.i32;
+
+  return res;
 }
 
 static void
@@ -227,12 +248,13 @@ line_decode(const struct line* line_encoded, struct shtr_line* line)
   line->lower_state_energy = line_encoded->lower_state_energy;
   line->delta_air = line_encoded->delta_air;
 
-  /* Convert gamma_air and gamma_self from fixed-point numbers (0:14) to
-   * double-precision floating-point numbers */
+  /* Convert gamma_air and gamma_self from fixed-point numbers on 14 bits to
+   * double-precision floating-point numbers. Gamma air is encode un (0:14) and
+   * gamma self in (3:11) */
   i64 = (line_encoded->gair14_gself14_isoid4 >> 18) & (BIT_I32(14)-1);
   line->gamma_air = fix2float(i64, 0, 14);
   i64 = (line_encoded->gair14_gself14_isoid4 >> 4) & (BIT_I32(14)-1);
-  line->gamma_self = fix2float(i64, 0, 14);
+  line->gamma_self = fix2float(i64, 3, 11);
 
   /* Unpack the isotope ID */
   line->isotope_id_local = line_encoded->gair14_gself14_isoid4 & (BIT_I32(4)-1);
@@ -262,7 +284,16 @@ register_line
   /* Pre-conditions */
   ASSERT(list && txtrdr && line);
 
-  line_encode(line, &ln_encoded);
+  res = line_encode(list->shtr, txtrdr, line, &ln_encoded);
+  switch(res) {
+    case RES_OK: /* nothing to do */; break;
+    case RES_BAD_OP:
+      WARN(list->shtr, "%s:%lu: line discarded\n",
+        txtrdr_get_name(txtrdr), txtrdr_get_line_num(txtrdr));
+      res = RES_OK;
+      goto exit;
+    default: goto error;
+  }
 
   /* Check if a line has been saved. If so, ensure that the lines are sorted */
   if(list->nlines) {