goat_cli/utils/
url.rs

1use crate::error::Result;
2use crate::{
3    utils::{
4        expression::CLIexpression,
5        field_registry,
6        variable_data::{GOAT_ASSEMBLY_VARIABLE_DATA, GOAT_TAXON_VARIABLE_DATA},
7        variables::Variables,
8    },
9    IndexType,
10};
11use url::Url;
12
13// format the ranks for the URL.
14
15/// Function to format the rank into a plain comma-separated string for the `ranks` URL parameter.
16///
17/// Returns an empty string if the rank is not recognised (e.g. "none").
18fn format_rank(r: &str) -> String {
19    let ranks = [
20        "subspecies",
21        "species",
22        "genus",
23        "family",
24        "order",
25        "class",
26        "phylum",
27        "kingdom",
28        "superkingdom",
29    ];
30    let position_selected = ranks.iter().position(|e| e == &r);
31    match position_selected {
32        Some(p) => ranks[p..].join(","),
33        None => "".to_string(),
34    }
35}
36
37/// Percent-encode a URL query parameter value using `%20` for spaces.
38///
39/// Unlike `url::form_urlencoded` (which encodes spaces as `+`), this produces
40/// standard URL percent-encoding that GoaT's API requires.
41pub fn percent_encode_query_value(s: &str) -> String {
42    let mut encoded = String::with_capacity(s.len() * 3);
43    for byte in s.bytes() {
44        match byte {
45            // RFC 3986 unreserved characters — pass through unchanged
46            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
47                encoded.push(byte as char);
48            }
49            // Everything else (including space → %20, < → %3C, ( → %28, etc.)
50            b => encoded.push_str(&format!("%{:02X}", b)),
51        }
52    }
53    encoded
54}
55
56/// Returns the plain comma-separated value for the `names` URL parameter, or empty string.
57fn format_names(flag: bool) -> String {
58    match flag {
59        true => "synonym,tol_id,common_name".to_string(),
60        false => "".to_string(),
61    }
62}
63
64/// Format an expression put into the `-e` flag on the CLI.
65pub fn format_expression(exp: &str, index_type: IndexType) -> Result<String> {
66    // Pull any live fields from the dynamic registry (empty if not yet initialised).
67    let dynamic_fields = field_registry::get_all_fields(index_type);
68    let extra = if dynamic_fields.is_empty() {
69        None
70    } else {
71        Some(&dynamic_fields)
72    };
73    let mut new_exp = CLIexpression::new(exp);
74    let parsed_string = match index_type {
75        IndexType::Taxon => new_exp.parse(&GOAT_TAXON_VARIABLE_DATA, extra)?,
76        IndexType::Assembly => new_exp.parse(&GOAT_ASSEMBLY_VARIABLE_DATA, extra)?,
77    };
78    Ok(parsed_string)
79}
80
81/// Boolean struct containing all of the CLI flag information
82/// passed from the user. This struct has been expanded to include
83/// both `taxon` and `assembly` indexes.
84#[derive(Copy, Clone, Debug, Default)]
85pub struct FieldBuilder {
86    /// Add only assembly level/span GoaT fields.
87    ///
88    /// A taxon index flag.
89    pub taxon_assembly: bool,
90    /// Add bioproject GoaT field.
91    ///
92    /// A taxon index flag.
93    pub taxon_bioproject: bool,
94    /// Add BUSCO completeness.
95    ///
96    /// A taxon index flag.
97    pub taxon_busco: bool,
98    /// Add country list GoaT field.
99    ///
100    /// A taxon index flag.
101    pub taxon_country_list: bool,
102    /// Add C-value information GoaT field.
103    ///
104    /// A taxon index flag.
105    pub taxon_cvalues: bool,
106    /// Add assembly & EBP metric date GoaT fields.
107    ///
108    /// A taxon index flag.
109    pub taxon_date: bool,
110    /// Add GC percent GoaT field.
111    ///
112    /// A taxon index flag.
113    pub taxon_gc_percent: bool,
114    /// Add gene count GoaT field.
115    ///
116    /// A taxon index flag.
117    pub taxon_gene_count: bool,
118    /// Add genome size GoaT fields.
119    ///
120    /// A taxon index flag.
121    pub taxon_gs: bool,
122    /// Add karyotype GoaT fields; chromosome number and
123    /// haploid number.
124    ///
125    /// A taxon index flag.
126    pub taxon_karyotype: bool,
127    /// Add return information for `isb_wildlife_act_1976`,
128    /// `habreg_2017`, `marhabreg-2017`, `waca_1981`,
129    /// `protection_of_badgers_act_1992`, `echabs92`
130    ///
131    /// A taxon index flag.
132    pub taxon_legislation: bool,
133    /// Add mitochondrial assembly span and gc percent
134    /// GoaT fields.
135    ///
136    /// A taxon index flag.
137    pub taxon_mitochondrion: bool,
138    /// Add contig and scaffold n50 GoaT fields.
139    ///
140    /// A taxon index flag.
141    pub taxon_n50: bool,
142    /// Add synonym, tolID, and common name GoaT fields.
143    ///
144    /// Not implemented in [`FieldBuilder`] below.
145    ///
146    /// A taxon index flag.
147    pub taxon_names: bool,
148    /// Add plastid assembly span and gc percent GoaT
149    /// fields.
150    ///
151    /// A taxon index flag.
152    pub taxon_plastid: bool,
153    /// Add ploidy GoaT field.
154    ///
155    /// A taxon index flag.
156    pub taxon_ploidy: bool,
157    /// Add sex determination GoaT field.
158    ///
159    /// A taxon index flag.
160    pub taxon_sex_determination: bool,
161    /// Add sample tracking information GoaT field.
162    ///
163    /// A taxon index flag.
164    pub taxon_status: bool,
165    /// Add `long_list`, `other_priority`, and `family_representative`
166    /// GoaT fields.
167    ///
168    /// A taxon index flag.
169    pub taxon_target_lists: bool,
170    /// Render output in tidy format?
171    ///
172    /// Not implemented in [`FieldBuilder`] below.
173    ///
174    /// A taxon index flag.
175    pub taxon_tidy: bool,
176    /// For each variable, show each of the direct/ancestor/descendent
177    /// as separate columns
178    pub taxon_toggle_direct: bool,
179    /// Assembly span and level.
180    ///
181    /// An assembly index flag.
182    pub assembly_assembly: bool,
183    /// Only chromosome count.
184    ///
185    /// An assembly index flag.
186    pub assembly_karyotype: bool,
187    /// All the contig information.
188    ///
189    /// An assembly index flag.
190    pub assembly_contig: bool,
191    /// All scaffold information.
192    ///
193    /// An assembly index flag.
194    pub assembly_scaffold: bool,
195    /// GC content.
196    ///
197    /// An assembly index flag.
198    pub assembly_gc: bool,
199    /// Gene and non-coding gene count.
200    ///
201    /// An assembly index flag.
202    pub assembly_gene: bool,
203    /// BUSCO completeness, lineage and string.
204    ///
205    /// An assembly index flag.
206    pub assembly_busco: bool,
207    /// BlobToolKit stats(?). No hit/target.
208    ///
209    /// An assembly index flag.
210    pub assembly_btk: bool,
211}
212
213impl FieldBuilder {
214    /// A function to turn all of the fields into a small data structure.
215    ///
216    /// This is hardcoded, but could be modified to be read in from
217    /// the goat standard variables JSON in the future.
218    ///
219    /// It's a [`Vec`] of a tuple of:
220    /// - [`bool`] which shows whether the user chose this flag or not
221    /// - [`Vec<&str>`] which enumerates the variable strings (as GoaT
222    /// would recognise) that correspond to this field.
223    ///
224    /// It's a bit of a judgement call on my part but happy to change if
225    /// there is a compelling argument.
226    fn to_vec_tuples(&self) -> Vec<(bool, Vec<&str>)> {
227        vec![
228            // Add all of the taxon_* fields
229            (self.taxon_assembly, vec!["assembly_level", "assembly_span"]),
230            (self.taxon_bioproject, vec!["bioproject", "biosample"]),
231            // testing all these busco fields.
232            (
233                self.taxon_busco,
234                vec![
235                    "busco_completeness",
236                    "odb10_lineage",
237                    "busco_lineage",
238                    "busco_string",
239                ],
240            ),
241            (self.taxon_country_list, vec!["country_list"]),
242            (self.taxon_cvalues, vec!["c_value"]),
243            (self.taxon_date, vec!["assembly_date", "ebp_metric_date"]),
244            (self.taxon_gc_percent, vec!["gc_percent"]),
245            (self.taxon_gene_count, vec!["gene_count"]),
246            (
247                self.taxon_gs,
248                vec!["genome_size", "genome_size_kmer", "genome_size_draft"],
249            ),
250            (
251                self.taxon_karyotype,
252                vec!["chromosome_number", "haploid_number"],
253            ),
254            (
255                self.taxon_legislation,
256                vec![
257                    "isb_wildlife_act_1976",
258                    "HabReg_2017",
259                    "MarHabReg-2017",
260                    "waca_1981",
261                    "Protection_of_Badgers_Act_1992",
262                    "ECHabs92",
263                ],
264            ),
265            (
266                self.taxon_mitochondrion,
267                vec!["mitochondrion_assembly_span", "mitochondrion_gc_percent"],
268            ),
269            (self.taxon_n50, vec!["scaffold_n50", "contig_n50"]),
270            (
271                self.taxon_plastid,
272                vec!["plastid_assembly_span", "plastid_gc_percent"],
273            ),
274            (self.taxon_ploidy, vec!["ploidy"]),
275            (self.taxon_sex_determination, vec!["sex_determination"]),
276            // there's now a bunch of sequencing status_asg/b10k/cbp... etc
277            // don't know if these should go here.
278            (
279                self.taxon_status,
280                vec![
281                    "sequencing_status",
282                    "sample_collected",
283                    "sample_acquired",
284                    "in_progress",
285                    "insdc_submitted",
286                    "insdc_open",
287                    "published",
288                    "sample_collected_by",
289                ],
290            ),
291            (
292                self.taxon_target_lists,
293                vec!["long_list", "other_priority", "family_representative"],
294            ),
295            // Add all of the assembly_* fields
296            (
297                self.assembly_assembly,
298                vec!["assembly_level", "assembly_span"],
299            ),
300            (self.assembly_btk, vec!["nohit", "target"]),
301            (
302                self.assembly_busco,
303                vec!["busco_completeness", "busco_lineage", "busco_string"],
304            ),
305            (
306                self.assembly_contig,
307                vec!["contig_count", "contig_l50", "contig_n50"],
308            ),
309            (self.assembly_gc, vec!["gc_percent"]),
310            (
311                self.assembly_gene,
312                vec!["gene_count", "noncoding_gene_count"],
313            ),
314            (self.assembly_karyotype, vec!["chromosome_count"]),
315            (
316                self.assembly_scaffold,
317                vec!["scaffold_count", "scaffold_l50", "scaffold_n50"],
318            ),
319        ]
320    }
321
322    /// Returns a plain comma-separated field list (no `&fields=` prefix, no percent-encoding).
323    ///
324    /// Returns an empty string if no field flags are set.
325    pub fn build_fields_string(&self) -> String {
326        let data = self.to_vec_tuples();
327
328        let mut fields: Vec<String> = Vec::new();
329        for (field_present, field_vec) in data.iter() {
330            if *field_present {
331                for field in field_vec {
332                    fields.push(field.to_string());
333                    if self.taxon_toggle_direct {
334                        fields.push(format!("{}:direct", field));
335                        fields.push(format!("{}:ancestor", field));
336                        fields.push(format!("{}:descendant", field));
337                    }
338                }
339            }
340        }
341
342        fields.join(",")
343    }
344
345    /// Returns key-value pairs for excluding missing and ancestral values.
346    ///
347    /// Each pair is `(param_name, field_name)` — the URL builder handles encoding.
348    fn generate_exclude_flags(&self) -> Vec<(String, String)> {
349        let data = self.to_vec_tuples();
350        let mut pairs: Vec<(String, String)> = Vec::new();
351
352        let mut exclude_index: usize = 0;
353        for (field_present, field_vec) in data.iter() {
354            if *field_present {
355                for field in field_vec {
356                    pairs.push((
357                        format!("excludeAncestral[{}]", exclude_index),
358                        field.to_string(),
359                    ));
360                    pairs.push((
361                        format!("excludeMissing[{}]", exclude_index),
362                        field.to_string(),
363                    ));
364                    exclude_index += 1;
365                }
366            }
367        }
368
369        pairs
370    }
371}
372
373/// Combine the plain field lists from the `-v` flag and the field-builder flags.
374///
375/// Both inputs are plain comma-separated lists (no prefix, no encoding).
376fn combine_variable_string(v: String, fb: String) -> String {
377    match (v.is_empty(), fb.is_empty()) {
378        (true, true) => "".into(),
379        (true, false) => fb,
380        (false, true) => v,
381        (false, false) => format!("{},{}", v, fb),
382    }
383}
384
385/// Build the `query` value from its parts.
386///
387/// An empty `taxon` makes a query across all taxa. `tax_rank` and
388/// `expression` are as returned by [`crate::utils::tax_ranks::TaxRanks::parse`]
389/// and [`format_expression`], i.e. starting with `" AND "`. As `OR` binds
390/// more loosely than `AND` in the API, the taxon and rank are repeated in
391/// each `OR` branch of the expression, so they apply to all of them.
392pub fn build_query(taxon: &str, tax_tree: &str, tax_rank: &str, expression: &str) -> String {
393    let taxon_term = if taxon.is_empty() {
394        String::new()
395    } else {
396        format!("tax_{}({})", tax_tree, taxon)
397    };
398    let tax_rank = tax_rank.trim_start_matches(" AND ");
399    let expression = expression.trim_start_matches(" AND ");
400
401    expression
402        .split(" OR ")
403        .map(|group| {
404            [taxon_term.as_str(), tax_rank, group]
405                .into_iter()
406                .filter(|part| !part.is_empty())
407                .collect::<Vec<_>>()
408                .join(" AND ")
409        })
410        .collect::<Vec<_>>()
411        .join(" OR ")
412}
413
414/// The `sortBy`/`sortOrder` URL parameters (with a leading `&`) for
415/// `--sort-by`/`--sort-order`.
416///
417/// GoaT ignores a field it doesn't know, so the field is checked here: it
418/// must be a field of the index (synonyms are resolved) or one of the
419/// record's own fields.
420pub fn sort_params(sort_by: &str, order: &str, index_type: IndexType) -> Result<String> {
421    let builtin: &[&str] = match index_type {
422        IndexType::Taxon => &["scientific_name", "taxon_id", "taxon_rank"],
423        IndexType::Assembly => &["scientific_name", "taxon_id", "assembly_id"],
424    };
425    let field = sort_by.trim();
426    let field = if builtin.contains(&field) {
427        field.to_string()
428    } else {
429        let data = match index_type {
430            IndexType::Taxon => &*GOAT_TAXON_VARIABLE_DATA,
431            IndexType::Assembly => &*GOAT_ASSEMBLY_VARIABLE_DATA,
432        };
433        Variables::new(field).parse_one(data)?
434    };
435    Ok(format!(
436        "&sortBy={}&sortOrder={}",
437        percent_encode_query_value(&field),
438        order
439    ))
440}
441
442/// The function which creats the GoaT API URLs
443/// which are then used as GET requests.
444pub fn make_goat_urls(
445    api: &str,
446    taxids: &[String],
447    goat_url: &str,
448    tax_tree: &str,
449    include_estimates: bool,
450    include_raw_values: bool,
451    exclude: bool,
452    summarise_values_by: &str,
453    result: &str,
454    taxonomy: &str,
455    size: u64,
456    ranks: &str,
457    fields: FieldBuilder,
458    variables: Option<&str>,
459    expression: &str,
460    tax_rank: &str,
461    unique_ids: Vec<String>,
462    index_type: IndexType,
463) -> Result<Vec<String>> {
464    // make the rank string
465    let rank_string = format_rank(ranks);
466
467    // parse the variables, if they have been given.
468    let variables_field_string = if let Some(variables) = variables {
469        match index_type {
470            IndexType::Taxon => Variables::new(variables)
471                .parse(&GOAT_TAXON_VARIABLE_DATA, fields.taxon_toggle_direct)?,
472            IndexType::Assembly => Variables::new(variables)
473                .parse(&GOAT_ASSEMBLY_VARIABLE_DATA, fields.taxon_toggle_direct)?,
474        }
475    } else {
476        "".into()
477    };
478
479    // parse the fields from the flags
480    let field_builder_string = fields.build_fields_string();
481    // and combine
482    let fields_string = combine_variable_string(variables_field_string, field_builder_string);
483
484    let names_string = format_names(fields.taxon_names);
485
486    let exclude_pairs: Vec<(String, String)> = if exclude {
487        match variables {
488            Some(v) => match index_type {
489                IndexType::Taxon => Variables::new(v).parse_exclude(&GOAT_TAXON_VARIABLE_DATA)?,
490                IndexType::Assembly => {
491                    Variables::new(v).parse_exclude(&GOAT_ASSEMBLY_VARIABLE_DATA)?
492                }
493            },
494            None => fields.generate_exclude_flags(),
495        }
496    } else {
497        vec![]
498    };
499
500    // enumeration of the taxa will be 0 -> n,
501    // corresponding to alphabetical order of taxa
502    let mut res = Vec::new();
503    for (taxon, query_id_suffix) in taxids.iter().zip(unique_ids.iter()) {
504        // Build the GoaT query language value (plain text; encoded below)
505        let query_value = build_query(taxon, tax_tree, tax_rank, expression);
506
507        // Build the raw query string manually so that spaces are encoded as
508        // %20 (not +).  url::query_pairs_mut uses form-encoding which GoaT rejects.
509        let mut qp: Vec<String> = Vec::new();
510        qp.push(format!("query={}", percent_encode_query_value(&query_value)));
511        qp.push(format!("includeEstimates={}", include_estimates));
512        qp.push(format!("includeRawValues={}", include_raw_values));
513        qp.push(format!("summaryValues={}", summarise_values_by));
514        qp.push(format!("result={}", result));
515        qp.push(format!("taxonomy={}", taxonomy));
516        qp.push(format!("size={}", size));
517        if !rank_string.is_empty() {
518            qp.push(format!("ranks={}", percent_encode_query_value(&rank_string)));
519        }
520        if !fields_string.is_empty() {
521            qp.push(format!("fields={}", percent_encode_query_value(&fields_string)));
522        }
523        if fields.taxon_tidy {
524            qp.push("tidyData=true".into());
525        }
526        if !names_string.is_empty() {
527            qp.push(format!("names={}", percent_encode_query_value(&names_string)));
528        }
529        qp.push(format!("queryId=goat_cli_{}", query_id_suffix));
530        for (key, value) in &exclude_pairs {
531            qp.push(format!("{}={}", key, percent_encode_query_value(value)));
532        }
533
534        let base = format!("{}{}", goat_url, api);
535        let mut url = Url::parse(&base).expect("goat_url is a valid base");
536        url.set_query(Some(&qp.join("&")));
537        res.push(url.to_string());
538    }
539    Ok(res)
540}