SeQuiLa | sequila

BAM

Mon, 01 Jan 0001 00:00:00 +0000

Examples

mkdir -p /tmp/data
wget -nc https://github.com/biodatageeks/pysequila/raw/master/features/data/NA12878.multichrom.md.bam -O /tmp/data/NA12878.bam
wget -nc https://github.com/biodatageeks/pysequila/raw/master/features/data/NA12878.multichrom.md.bam.bai -O /tmp/data/NA12878.bam.bai

CREATE TABLE IF NOT EXISTS reads
USING org.biodatageeks.sequila.datasources.BAM.BAMDataSource
OPTIONS(path "/tmp/data/NA12878.bam");

DESCRIBE TABLE reads;

+---------+---------+-------+
| col_name|data_type|comment|
+---------+---------+-------+
|sample_id|   string|   null|
|    qname|   string|   null|
|     flag|      int|   null|
|   contig|   string|   null|
|      pos|      int|   null|
|pos_start|      int|   null|
|  pos_end|      int|   null|
|     mapq|      int|   null|
|    cigar|   string|   null|
|    rnext|   string|   null|
|    pnext|      int|   null|
|     tlen|      int|   null|
|      seq|   string|   null|
|     qual|   string|   null|
|   tag_AM|      int|   null|
|   tag_AS|      int|   null|
|   tag_BC|   string|   null|
|   tag_BQ|   string|   null|
|   tag_BZ|   string|   null|
|   tag_CB|   string|   null|
|   tag_CC|   string|   null|
|   tag_CG|   string|   null|
|   tag_CM|      int|   null|
|   tag_CO|   string|   null|
|   tag_CP|      int|   null|
|   tag_CQ|   string|   null|
|   tag_CR|   string|   null|
|   tag_CS|   string|   null|
|   tag_CT|   string|   null|
|   tag_CY|   string|   null|
|   tag_E2|   string|   null|
|   tag_FI|      int|   null|
|   tag_FS|   string|   null|
|   tag_FZ|   string|   null|
|   tag_H0|      int|   null|
|   tag_H1|      int|   null|
|   tag_H2|      int|   null|
|   tag_HI|      int|   null|
|   tag_IH|      int|   null|
|   tag_LB|   string|   null|
|   tag_MC|   string|   null|
|   tag_MD|   string|   null|
|   tag_MI|   string|   null|
|   tag_MQ|      int|   null|
|   tag_NH|      int|   null|
|   tag_NM|      int|   null|
|   tag_OA|   string|   null|
|   tag_OC|   string|   null|
|   tag_OP|      int|   null|
|   tag_OQ|   string|   null|
|   tag_OX|   string|   null|
|   tag_PG|   string|   null|
|   tag_PQ|      int|   null|
|   tag_PT|   string|   null|
|   tag_PU|   string|   null|
|   tag_Q2|   string|   null|
|   tag_QT|   string|   null|
|   tag_QX|   string|   null|
|   tag_R2|   string|   null|
|   tag_RG|   string|   null|
|   tag_RX|   string|   null|
|   tag_SA|   string|   null|
|   tag_SM|      int|   null|
|   tag_TC|      int|   null|
|   tag_U2|   string|   null|
|   tag_UQ|      int|   null|
+---------+---------+-------+

SELECT qname, contig, pos_start, pos_end, cigar, seq FROM reads LIMIT 5;

+--------------------+------+---------+-------+-----+--------------------+
|               qname|contig|pos_start|pos_end|cigar|                 seq|
+--------------------+------+---------+-------+-----+--------------------+
|61DC0AAXX100127:8...|    MT|        7|    107| 101M|AGGTCTATCACCCTATT...|
|61DC0AAXX100127:8...|    MT|        9|    109| 101M|GTCTGTCACCCTTGTAG...|
|61DC0AAXX100127:8...|    MT|       10|    110| 101M|TCTATCCCCCTATTAAC...|
|61DC0AAXX100127:8...|    MT|       20|    120| 101M|TATTATCCACTCACGGG...|
|61CC3AAXX100125:5...|    MT|       25|    100|  76M|ACCACTCACGGGAGCTC...|
+--------------------+------+---------+-------+-----+--------------------+

SELECT count(*) AS cnt FROM reads;

+-----+
|  cnt|
+-----+
|22607|
+-----+

>>> from pysequila import SequilaSession
>>> ss = SequilaSession \
.builder \
.getOrCreate()

>>> reads_df=ss.read\
.format("org.biodatageeks.sequila.datasources.BAM.BAMDataSource")\
.load("/tmp/data/NA12878.bam")
>>> reads_df.printSchema()
root
|-- sample_id: string (nullable = true)
|-- qname: string (nullable = true)
|-- flag: integer (nullable = false)
|-- contig: string (nullable = true)
|-- pos: integer (nullable = false)
|-- pos_start: integer (nullable = false)
|-- pos_end: integer (nullable = false)
|-- mapq: integer (nullable = false)
|-- cigar: string (nullable = true)
|-- rnext: string (nullable = true)
|-- pnext: integer (nullable = false)
|-- tlen: integer (nullable = false)
|-- seq: string (nullable = true)
|-- qual: string (nullable = true)
|-- tag_AM: integer (nullable = true)
|-- tag_AS: integer (nullable = true)
|-- tag_BC: string (nullable = true)
|-- tag_BQ: string (nullable = true)
|-- tag_BZ: string (nullable = true)
|-- tag_CB: string (nullable = true)
|-- tag_CC: string (nullable = true)
|-- tag_CG: string (nullable = true)
|-- tag_CM: integer (nullable = true)
|-- tag_CO: string (nullable = true)
|-- tag_CP: integer (nullable = true)
|-- tag_CQ: string (nullable = true)
|-- tag_CR: string (nullable = true)
|-- tag_CS: string (nullable = true)
|-- tag_CT: string (nullable = true)
|-- tag_CY: string (nullable = true)
|-- tag_E2: string (nullable = true)
|-- tag_FI: integer (nullable = true)
|-- tag_FS: string (nullable = true)
|-- tag_FZ: string (nullable = true)
|-- tag_H0: integer (nullable = true)
|-- tag_H1: integer (nullable = true)
|-- tag_H2: integer (nullable = true)
|-- tag_HI: integer (nullable = true)
|-- tag_IH: integer (nullable = true)
|-- tag_LB: string (nullable = true)
|-- tag_MC: string (nullable = true)
|-- tag_MD: string (nullable = true)
|-- tag_MI: string (nullable = true)
|-- tag_MQ: integer (nullable = true)
|-- tag_NH: integer (nullable = true)
|-- tag_NM: integer (nullable = true)
|-- tag_OA: string (nullable = true)
|-- tag_OC: string (nullable = true)
|-- tag_OP: integer (nullable = true)
|-- tag_OQ: string (nullable = true)
|-- tag_OX: string (nullable = true)
|-- tag_PG: string (nullable = true)
|-- tag_PQ: integer (nullable = true)
|-- tag_PT: string (nullable = true)
|-- tag_PU: string (nullable = true)
|-- tag_Q2: string (nullable = true)
|-- tag_QT: string (nullable = true)
|-- tag_QX: string (nullable = true)
|-- tag_R2: string (nullable = true)
|-- tag_RG: string (nullable = true)
|-- tag_RX: string (nullable = true)
|-- tag_SA: string (nullable = true)
|-- tag_SM: integer (nullable = true)
|-- tag_TC: integer (nullable = true)
|-- tag_U2: string (nullable = true)
|-- tag_UQ: integer (nullable = true)

>>> reads_df.select("qname", "contig", "pos_start", "pos_end", "cigar", "seq" ).show(5)
+--------------------+------+---------+-------+-----+--------------------+
|               qname|contig|pos_start|pos_end|cigar|                 seq|
+--------------------+------+---------+-------+-----+--------------------+
|61DC0AAXX100127:8...|    MT|        7|    107| 101M|AGGTCTATCACCCTATT...|
|61DC0AAXX100127:8...|    MT|        9|    109| 101M|GTCTGTCACCCTTGTAG...|
|61DC0AAXX100127:8...|    MT|       10|    110| 101M|TCTATCCCCCTATTAAC...|
|61DC0AAXX100127:8...|    MT|       20|    120| 101M|TATTATCCACTCACGGG...|
|61CC3AAXX100125:5...|    MT|       25|    100|  76M|ACCACTCACGGGAGCTC...|
+--------------------+------+---------+-------+-----+--------------------+
only showing top 5 rows

>>> reads_df.count()
22607

CRAM

Mon, 01 Jan 0001 00:00:00 +0000

Examples

mkdir -p /tmp/data
wget -nc https://github.com/biodatageeks/pysequila/raw/master/features/data/NA12878.multichrom.md.cram -O /tmp/data/NA12878.cram
wget -nc https://github.com/biodatageeks/pysequila/raw/master/features/data/NA12878.multichrom.md.cram.crai -O /tmp/data/NA12878.cram.crai

wget -nc https://github.com/biodatageeks/pysequila/raw/master/features/data/Homo_sapiens_assembly18_chr1_chrM.small.fasta -O /tmp/data/hg18.fasta
wget -nc https://github.com/biodatageeks/pysequila/raw/master/features/data/Homo_sapiens_assembly18_chr1_chrM.small.fasta.fai -O /tmp/data/hg18.fasta.fai

CREATE TABLE IF NOT EXISTS reads
USING org.biodatageeks.sequila.datasources.BAM.CRAMDataSource
OPTIONS(path "/tmp/data/NA12878.cram", refPath "/tmp/data/hg18.fasta");

DESCRIBE TABLE reads;

+---------+---------+-------+
| col_name|data_type|comment|
+---------+---------+-------+
|sample_id|   string|   null|
|    qname|   string|   null|
|     flag|      int|   null|
|   contig|   string|   null|
|      pos|      int|   null|
|pos_start|      int|   null|
|  pos_end|      int|   null|
|     mapq|      int|   null|
|    cigar|   string|   null|
|    rnext|   string|   null|
|    pnext|      int|   null|
|     tlen|      int|   null|
|      seq|   string|   null|
|     qual|   string|   null|
|   tag_AM|      int|   null|
|   tag_AS|      int|   null|
|   tag_BC|   string|   null|
|   tag_BQ|   string|   null|
|   tag_BZ|   string|   null|
|   tag_CB|   string|   null|
|   tag_CC|   string|   null|
|   tag_CG|   string|   null|
|   tag_CM|      int|   null|
|   tag_CO|   string|   null|
|   tag_CP|      int|   null|
|   tag_CQ|   string|   null|
|   tag_CR|   string|   null|
|   tag_CS|   string|   null|
|   tag_CT|   string|   null|
|   tag_CY|   string|   null|
|   tag_E2|   string|   null|
|   tag_FI|      int|   null|
|   tag_FS|   string|   null|
|   tag_FZ|   string|   null|
|   tag_H0|      int|   null|
|   tag_H1|      int|   null|
|   tag_H2|      int|   null|
|   tag_HI|      int|   null|
|   tag_IH|      int|   null|
|   tag_LB|   string|   null|
|   tag_MC|   string|   null|
|   tag_MD|   string|   null|
|   tag_MI|   string|   null|
|   tag_MQ|      int|   null|
|   tag_NH|      int|   null|
|   tag_NM|      int|   null|
|   tag_OA|   string|   null|
|   tag_OC|   string|   null|
|   tag_OP|      int|   null|
|   tag_OQ|   string|   null|
|   tag_OX|   string|   null|
|   tag_PG|   string|   null|
|   tag_PQ|      int|   null|
|   tag_PT|   string|   null|
|   tag_PU|   string|   null|
|   tag_Q2|   string|   null|
|   tag_QT|   string|   null|
|   tag_QX|   string|   null|
|   tag_R2|   string|   null|
|   tag_RG|   string|   null|
|   tag_RX|   string|   null|
|   tag_SA|   string|   null|
|   tag_SM|      int|   null|
|   tag_TC|      int|   null|
|   tag_U2|   string|   null|
|   tag_UQ|      int|   null|
+---------+---------+-------+

SELECT qname, contig, pos_start, pos_end, cigar, seq FROM reads LIMIT 5;

+--------------------+------+---------+-------+-----+--------------------+
|               qname|contig|pos_start|pos_end|cigar|                 seq|
+--------------------+------+---------+-------+-----+--------------------+
|61DC0AAXX100127:8...|    MT|        7|    107| 101M|AGGTCTATCACCCTATT...|
|61DC0AAXX100127:8...|    MT|        9|    109| 101M|GTCTGTCACCCTTGTAG...|
|61DC0AAXX100127:8...|    MT|       10|    110| 101M|TCTATCCCCCTATTAAC...|
|61DC0AAXX100127:8...|    MT|       20|    120| 101M|TATTATCCACTCACGGG...|
|61CC3AAXX100125:5...|    MT|       25|    100|  76M|ACCACTCACGGGAGCTC...|
+--------------------+------+---------+-------+-----+--------------------+

SELECT count(*) AS cnt FROM reads;

+-----+
|  cnt|
+-----+
|22607|
+-----+

>>> from pysequila import SequilaSession
>>> ss = SequilaSession \
.builder \
.getOrCreate()

>>> reads_df=ss.read\
.format("org.biodatageeks.sequila.datasources.BAM.CRAMDataSource")\
.option("refPath","/tmp/data/hg18.fasta") \
.load("/tmp/data/NA12878.cram")

>>> reads_df.printSchema()
root
|-- sample_id: string (nullable = true)
|-- qname: string (nullable = true)
|-- flag: integer (nullable = false)
|-- contig: string (nullable = true)
|-- pos: integer (nullable = false)
|-- pos_start: integer (nullable = false)
|-- pos_end: integer (nullable = false)
|-- mapq: integer (nullable = false)
|-- cigar: string (nullable = true)
|-- rnext: string (nullable = true)
|-- pnext: integer (nullable = false)
|-- tlen: integer (nullable = false)
|-- seq: string (nullable = true)
|-- qual: string (nullable = true)
|-- tag_AM: integer (nullable = true)
|-- tag_AS: integer (nullable = true)
|-- tag_BC: string (nullable = true)
|-- tag_BQ: string (nullable = true)
|-- tag_BZ: string (nullable = true)
|-- tag_CB: string (nullable = true)
|-- tag_CC: string (nullable = true)
|-- tag_CG: string (nullable = true)
|-- tag_CM: integer (nullable = true)
|-- tag_CO: string (nullable = true)
|-- tag_CP: integer (nullable = true)
|-- tag_CQ: string (nullable = true)
|-- tag_CR: string (nullable = true)
|-- tag_CS: string (nullable = true)
|-- tag_CT: string (nullable = true)
|-- tag_CY: string (nullable = true)
|-- tag_E2: string (nullable = true)
|-- tag_FI: integer (nullable = true)
|-- tag_FS: string (nullable = true)
|-- tag_FZ: string (nullable = true)
|-- tag_H0: integer (nullable = true)
|-- tag_H1: integer (nullable = true)
|-- tag_H2: integer (nullable = true)
|-- tag_HI: integer (nullable = true)
|-- tag_IH: integer (nullable = true)
|-- tag_LB: string (nullable = true)
|-- tag_MC: string (nullable = true)
|-- tag_MD: string (nullable = true)
|-- tag_MI: string (nullable = true)
|-- tag_MQ: integer (nullable = true)
|-- tag_NH: integer (nullable = true)
|-- tag_NM: integer (nullable = true)
|-- tag_OA: string (nullable = true)
|-- tag_OC: string (nullable = true)
|-- tag_OP: integer (nullable = true)
|-- tag_OQ: string (nullable = true)
|-- tag_OX: string (nullable = true)
|-- tag_PG: string (nullable = true)
|-- tag_PQ: integer (nullable = true)
|-- tag_PT: string (nullable = true)
|-- tag_PU: string (nullable = true)
|-- tag_Q2: string (nullable = true)
|-- tag_QT: string (nullable = true)
|-- tag_QX: string (nullable = true)
|-- tag_R2: string (nullable = true)
|-- tag_RG: string (nullable = true)
|-- tag_RX: string (nullable = true)
|-- tag_SA: string (nullable = true)
|-- tag_SM: integer (nullable = true)
|-- tag_TC: integer (nullable = true)
|-- tag_U2: string (nullable = true)
|-- tag_UQ: integer (nullable = true)

>>> reads_df.select("qname", "contig", "pos_start", "pos_end", "cigar", "seq" ).show(5)
+--------------------+------+---------+-------+-----+--------------------+
|               qname|contig|pos_start|pos_end|cigar|                 seq|
+--------------------+------+---------+-------+-----+--------------------+
|61DC0AAXX100127:8...|    MT|        7|    107| 101M|AGGTCTATCACCCTATT...|
|61DC0AAXX100127:8...|    MT|        9|    109| 101M|GTCTGTCACCCTTGTAG...|
|61DC0AAXX100127:8...|    MT|       10|    110| 101M|TCTATCCCCCTATTAAC...|
|61DC0AAXX100127:8...|    MT|       20|    120| 101M|TATTATCCACTCACGGG...|
|61CC3AAXX100125:5...|    MT|       25|    100|  76M|ACCACTCACGGGAGCTC...|
+--------------------+------+---------+-------+-----+--------------------+
only showing top 5 rows

>>> reads_df.count()
22607

General

Mon, 01 Jan 0001 00:00:00 +0000

Property Name	Default	Meaning
spark.biodatageeks.readAligment.method	hadoopBAM	Whether to use `hadoopBAM` or `disq`. Hadoop-BAM seems to be more performant in local mode but has worse horizontal scalability characteristic(~50+ executors).
spark.biodatageeks.bam.useGKLInflate	true	Whether to use Intel GKL deflater when using HadoopBAM (for disq it is always on)
spark.biodatageeks.bam.validation	SILENT	Method of BAM records validation

Pileup and coverage

Mon, 01 Jan 0001 00:00:00 +0000

Stay tuned for our new research paper Cloud-native distributed genomic pileup operations that will be published in January 2022!

Pileup and coverage

Mon, 01 Jan 0001 00:00:00 +0000

Testing environment

Datasets

Type	File	Size [GB]	Download
ES	NA12878.proper.wes.md.bam	17	BAM BAI SBI
WGS	NA12878.proper.wgs.md.bam	278	BAM BAI SBI
Reference	Homo_sapiens_assembly18.fasta	2.9	FASTA FAI

Tools

Software	Version	Pileup	Coverage	Multi-threaded	Distributed
ADAM	0.36.0	no	yes	yes	yes
GATK	4.2.3.0	yes	yes(only intervals)	no	no
GATK-Spark	4.2.3.0	yes	yes(only intervals)	yes	yes
megadepth	1.1.1	no	yes	yes(only I/O)	no
mosdepth	0.3.2	no	yes	yes(only I/O)	no
sambamba	0.8.1	no	yes	no	no
samtools	1.9	yes	yes	no	no
samtools	1.14	yes	yes	yes(only coverage, I/O)	no
SeQuiLa-cov	0.6.11	no	yes	yes	yes
SeQuiLa	1.0.0	yes	yes	yes	yes

Single node specification

Processor	Base freq [GHz]	CPUs	Total cores	Memory [GB]	OS Version	Disk
Intel(R) Xeon(R) E5-2618L v4	2.20	2	20(40)	256	RHEL 7.8(Maipo)	3TB(RAID1)

Hadoop cluster

Masters	Workers	Hadoop distribution	Total disk HDFS [TB]	Total YARN cores	Total YARN RAM [TB]	Net [Gbits]
6	34	HDP 3.1.4	700	1360(680)	6.8	100

SeQuiLa, ADAM and GATK(Spark) parameters

Parameter	Value	SeQuiLa only	Local-test	Cluster-test
spark.biodatageeks.pileup.useVectorizedOrcWriter	true	yes	yes	no
spark.sql.orc.compression.codec	snappy	no	yes	yes
spark.biodatageeks.readAligment.method	hadoopBAM	yes	yes	no
spark.biodatageeks.readAligment.method	disq	yes	no	yes
spark.serializer	org.apache.spark.serializer.KryoSerializer	yes¹	yes	yes
spark.kryo.registrator	org.biodatageeks.sequila.pileup.serializers.CustomKryoRegistrator	yes	yes	yes
spark.kryoserializer.buffer.max	1024m	yes	yes	yes
spark.hadoop.mapred.min.split.size	268435456	yes	no	yes
spark.hadoop.mapred.min.split.size	134217728	yes	yes	no
spark.driver.memory	8g	no	no	yes
spark.driver.memory	16g	no	yes	no
spark.executor.memory	4g²	no	yes	yes
spark.dynamicAllocation.enabled	false	no	yes	yes

Other parameters

Software	Operation	Command line options	Multithreading options
ADAM	coverage	`-- coverage -collapse`	`--spark-master local[$n]`
GATK	pileup	`PileupSpark`	`--spark-master local[$n]`
GATK	pileup	`Pileup`	–
megadepth	coverage	`--coverage --require-mdz --keep-order`	`--threads $(n-1)`
mosdepth	coverage	`-x`	`--threads $(n-1)`
sambamba	coverage	`depth base`	–nthreads=$n³
samtools	coverage	`depth`	`--threads $(n-1)`⁴
samtools	pileup	`mpileup -B -x -A -q 0 -Q 0`	–

SeQuiLa

Coverage

import org.apache.spark.sql.{SequilaSession, SparkSession}
val ss = SequilaSession(spark)
ss.sparkContext.setLogLevel("INFO")
val bamPath = "/scratch/wiewiom/WGS/NA12878.proper.wgs.md.bam"
val referencePath = "/home/wiewiom/data/Homo_sapiens_assembly18.fasta"
ss.time{
    ss
    .coverage(bamPath, referencePath)
    .write
    .orc("/tmp/sequila.coverage")
}

Pileup

import org.apache.spark.sql.{SequilaSession, SparkSession}
val ss = SequilaSession(spark)
ss.sparkContext.setLogLevel("INFO")
val bamPath = "/scratch/wiewiom/WGS/NA12878.proper.wgs.md.bam"
val referencePath = "/home/wiewiom/data/Homo_sapiens_assembly18.fasta"
ss.time{
    ss
    .pileup(bamPath, referencePath, true)
    .write
    .orc("/tmp/sequila.pileup")
}

Results single node

Coverage

Pileup

Results Hadoop cluster

Coverage

Pileup

GATK-Spark and ADAM also use fast serialization using Kryo lib and custom registrators. ↩︎
In case of GATK and WGS tests needed to be increased to 8g ↩︎
Does not result in increasing parallelism level ↩︎
Available in version >= 1.13 ↩︎

Python

Mon, 01 Jan 0001 00:00:00 +0000

sequila
- SequilaSession
  - Builder
  - coverage
  - pileup

Watchout

Check Configuration page for other configuration options that applicable for all types of APIs.

sequila

Entrypoint to Sequila - tool for large-scale genomics on Spark.

SequilaSession Objects

class SequilaSession(SparkSession)

Wrapper for SparkSession.

Builder Objects

class Builder(SparkSession.Builder)

Builder for :class:SequilaSession.

getOrCreate

def getOrCreate()

Get an existing :class:SequilaSession.

This is to add SequilaSession wrapper around SparkSession.

Examples

from pysequila import SequilaSession
ss = SequilaSession \
.builder \
.getOrCreate()

coverage

def coverage(path: str, refPath: str) -> DataFrame

Create a :class:DataFrame with depth of coverage for a specific aligment file.

.. versionadded:: 0.3.0

Parameters

Parameter Name	Type	Meaning
path	str	the alignment file in BAM/CRAM format (with an index file)
refPath	str	the reference file in FASTA format (with an index file)

Returns

:class:DataFrame

Examples

>>> ss.coverage(bam_file, ref_file).show(1)
+------+---------+-------+---+--------+
|contig|pos_start|pos_end|ref|coverage|
+------+---------+-------+---+--------+
|     1|       34|     34|  R|       1|
+------+---------+-------+---+--------+
only showing top 1 row

pileup

def pileup(path: str, refPath: str, qual: bool) -> DataFrame

Create a :class:DataFrame with pileup for a specific aligment file.

.. versionadded:: 0.3.0

Parameters

Parameter Name	Type	Meaning
path	str	the alignment file in BAM/CRAM format (with an index file)
refPath	str	the reference file in FASTA format (with an index file)
qual	bool(True)	whether to include base qualities pileup in the output

Returns

:class:DataFrame

Examples

>>> ss.pileup(bam_file, ref_file, True).where("alts IS NOT NULL").show(1)
+------+---------+-------+---+--------+--------+-----------+---------+--------------------+
|contig|pos_start|pos_end|ref|coverage|countRef|countNonRef|     alts|               quals|
+------+---------+-------+---+--------+--------+-----------+---------+--------------------+
|     1|       69|     69|  A|       7|       6|          1|{99 -> 1}|{65 -> [0, 0, 0, ...|
+------+---------+-------+---+--------+--------+-----------+---------+--------------------+
only showing top 1 row

SQL

Mon, 01 Jan 0001 00:00:00 +0000

Watchout

Check Configuration page for other configuration options that applicable for all types of APIs.

coverage

 coverage(table_name , sample_name, ref_path)

Parameters

Parameter Name	Type	Meaning
table_name	str	name of the external table created over alignment file(s) in BAM/CRAM format (with an index file - both BAI and SBI are supported)
sample_name	str	name of the sample in `table_name` to use for coverage calculations
ref_path	str	the reference file in FASTA format (with an index file)

Examples

CREATE TABLE IF NOT EXISTS reads
USING org.biodatageeks.sequila.datasources.BAM.BAMDataSource
OPTIONS(path "{{ PWD }}/data/NA12878.bam");
-- calculate coverage
SELECT * FROM coverage('reads', 'NA12878','{{ PWD }}/data/hg18.fasta') LIMIT 10;

+------+---------+-------+---+--------+
|contig|pos_start|pos_end|ref|coverage|
+------+---------+-------+---+--------+
|     1|       34|     34|  R|       1|
|     1|       35|     35|  R|       2|
|     1|       36|     37|  R|       3|
|     1|       38|     40|  R|       4|
|     1|       41|     49|  R|       5|
|     1|       50|     67|  R|       6|
|     1|       68|    109|  R|       7|
|     1|      110|    110|  R|       6|
|     1|      111|    111|  R|       5|
|     1|      112|    113|  R|       4|
+------+---------+-------+---+--------+
only showing top 10 rows

coverage

 pileup(table_name , sample_name, ref_path, include_quals)

Parameters

Parameter Name	Type	Meaning
table_name	str	name of the external table created over alignment file(s) in BAM/CRAM format (with an index file - both BAI and SBI are supported)
sample_name	str	name of the sample in `table_name` to use for coverage calculations
ref_path	str	the reference file in FASTA format (with an index file)
include_quals	bool(false)	whether to include base qualities pileup as well for positions with alts

Examples

CREATE TABLE IF NOT EXISTS reads
USING org.biodatageeks.sequila.datasources.BAM.BAMDataSource
OPTIONS(path "{{ PWD }}/data/NA12878.bam");
-- calculate pileup
SELECT * FROM pileup('reads', 'NA12878','{{ PWD }}/data/hg18.fasta', false) LIMIT 5;

+------+---------+-------+---------+--------+--------+-----------+----+-----+
|contig|pos_start|pos_end|      ref|coverage|countRef|countNonRef|alts|quals|
+------+---------+-------+---------+--------+--------+-----------+----+-----+
|     1|       34|     34|        C|       1|       1|          0|null| null|
|     1|       35|     35|        C|       2|       2|          0|null| null|
|     1|       36|     37|       CT|       3|       3|          0|null| null|
|     1|       38|     40|      AAC|       4|       4|          0|null| null|
|     1|       41|     49|CCTAACCCT|       5|       5|          0|null| null|
+------+---------+-------+---------+--------+--------+-----------+----+-----+
only showing top 5 rows

Pileup and coverage

Mon, 01 Jan 0001 00:00:00 +0000

Property Name	Default	Meaning
spark.biodatageeks.coverage.filterFlag	1796	Which SAM flags to use for filtering records. Check SAM explainer for details.
spark.biodatageeks.coverage.minMapQual	0	Minimal value of reads' mapping quality to be included in coverage/pileup calculations
spark.biodatageeks.pileup.useVectorizedOrcWriter	false	Whether to use vectorized ORC writer and to bypass Spark’s InternalRow serialization and write directly to the ouput file. This option is still experimental but speeds up saving results substantially.
spark.biodatageeks.pileup.maxBaseQuality	40	Specify the number of base quality values - internally used for allocating BQ counters.

BED

Mon, 01 Jan 0001 00:00:00 +0000

Examples

mkdir -p /tmp/data
wget -nc https://github.com/biodatageeks/pysequila/raw/master/features/data/targets.bed -O /tmp/data/targets.bed

CREATE TABLE IF NOT EXISTS targets
USING org.biodatageeks.sequila.datasources.BED.BEDDataSource
OPTIONS(path "/tmp/data/targets.bed");

DESCRIBE TABLE targets;

+------------+----------+-------+
|    col_name| data_type|comment|
+------------+----------+-------+
|      contig|    string|   null|
|   pos_start|       int|   null|
|     pos_end|       int|   null|
|        name|    string|   null|
|       score|       int|   null|
|      strand|    string|   null|
| thick_start|       int|   null|
|   thick_end|       int|   null|
|    item_rgb|array<int>|   null|
| block_count|       int|   null|
| block_sizes|array<int>|   null|
|block_starts|array<int>|   null|
+------------+----------+-------+

SELECT * FROM targets;

+------+---------+-------+----+-----+------+-----------+---------+--------+-----------+-----------+------------+
|contig|pos_start|pos_end|name|score|strand|thick_start|thick_end|item_rgb|block_count|block_sizes|block_starts|
+------+---------+-------+----+-----+------+-----------+---------+--------+-----------+-----------+------------+
|    MT|       11|     50|null| null|  null|       null|     null|    null|       null|       null|        null|
|    MT|      201|    300|null| null|  null|       null|     null|    null|       null|       null|        null|
+------+---------+-------+----+-----+------+-----------+---------+--------+-----------+-----------+------------+

>>> from pysequila import SequilaSession
>>> ss = SequilaSession \
.builder \
.getOrCreate()

>>> targets_df = ss.read\
.format("org.biodatageeks.sequila.datasources.BED.BEDDataSource")\
.load("/tmp/data/targets.bed")

>>> targets_df.printSchema()
root
|-- contig: string (nullable = true)
|-- pos_start: integer (nullable = false)
|-- pos_end: integer (nullable = false)
|-- name: string (nullable = true)
|-- score: integer (nullable = true)
|-- strand: string (nullable = true)
|-- thick_start: integer (nullable = true)
|-- thick_end: integer (nullable = true)
|-- item_rgb: array (nullable = true)
|    |-- element: integer (containsNull = false)
|-- block_count: integer (nullable = true)
|-- block_sizes: array (nullable = true)
|    |-- element: integer (containsNull = false)
|-- block_starts: array (nullable = true)
|    |-- element: integer (containsNull = false)

>>> targets_df.show()
+------+---------+-------+----+-----+------+-----------+---------+--------+-----------+-----------+------------+
|contig|pos_start|pos_end|name|score|strand|thick_start|thick_end|item_rgb|block_count|block_sizes|block_starts|
+------+---------+-------+----+-----+------+-----------+---------+--------+-----------+-----------+------------+
|    MT|       11|     50|null| null|  null|       null|     null|    null|       null|       null|        null|
|    MT|      201|    300|null| null|  null|       null|     null|    null|       null|       null|        null|
+------+---------+-------+----+-----+------+-----------+---------+--------+-----------+-----------+------------+

Interval joins

Mon, 01 Jan 0001 00:00:00 +0000

Overview

Consider datasets s1 and s2, storing genomic intervals, such as |s1|<|s2|. Let’s assume that their structure contains necessary genomic coordinates (chromosome, start position, end position) along with optional additional interval annotations (e.g. targetId).

Tables structure

Dataset `s1` and `s2`. Both storing genomic intervals with necessary genomic coordinates and optional annotations

Our goal is to efficiently execute a query as shown below:

    SELECT s2.targetId,count(*)
    FROM reads s1 JOIN targets s2
    ON s1.chr = s2.chr
    AND s1.end >= s2.start
    AND s1.start <= s2.end
    GROUP BY targetId;

Algorithm

At it’s core SeQuiLa’s range joins are based on IntervalTree data structure. The main idea of the algorithm is to transform dataset s1 into a broadcastable structure of an interval forest (a hash map of interval trees, each representing one chromosome). The intervals from dataset s2 can be efficiently intersected with the constructed interval forest.

An interval tree is a tree data structure to hold intervals. It is a augmented, balanced red-black tree with low endpoint as node key and additional max value of any endpoint stored in subtree. Each node contains following fields: parent, left subtree, right subtree, color, low endpoint, high endpoint and max endpoint of subtree. It can be proved that this structure allows for correct interval insertion, deletion and search in :math:O(lg n) time ([CLR]_)

Interval tree

An interval tree. On the top: A set of 10 intervals, shown sorted bottom to top by left endpoint. On the bottom the interval tree that represents them. An inorder tree walk of the tree lists the nodes in sorted order by left endpoint.

Our implementation of IntervalTree is based on explanations in [CLR]_ although it is extended in the following ways:

data structure allows storing non-unique intervals
data structure allows storing in the tree nodes additional interval annotations if they fit into dedicated Spark driver’s memory fraction

Let’s presume that we have a cluster with a Spark driver, three worker nodes and that the tables are partitioned between worker nodes. When interval query is performed, all dataset’s :math:s1 partitions are sent to the driver node on which interval forest is constructed (for each chromosome a separate interval tree is created). The forest is subsequently sent back to worker nodes on which efficient interval operations based on interval trees are performed. Depending on the strategy chosen by rule-based optimizer different set of columns are stored in tree nodes

Broadcasting mechanism

Broadcasting interval forest to worker nodes.

.. [CLR] Cormen, Thomas H.; Leiserson, Charles E., Rivest, Ronald L. (1990). Introduction to Algorithms (1st ed.). MIT Press and McGraw-Hill. ISBN 0-262-03141-8

Optimizations

SeQuiLa package introduces a new rule based optimizer (RBO) that chooses most efficient join strategy based on input data statistics computed in the runtime. The first step of the algorithm is to obtain value of maxBroadcastSize parameter. It can set explicitly by the end user or computed as a fraction of the Apache Spark Driver memory. In the next step table row counts are computed and based on that table with the fewer rows is selected for constructing interval forest. This is the default approach - it can be overridden by setting spark.biodatageeks.rangejoin.useJoinOrder to true. In this scenario no row counts are computed and the right join table is used for creating interval forest. Such an strategy can be useful in situation when it is known upfront which table should be used for creating a broadcast structure. The final step of the optimization procedure is to estimate the row size and the size of the whole projected table. If it fits into dedicated Spark Driver’s memory (controlled by maxBroadcastSize parameter) the interval forest is augmented with all columns from s1 (SeQuiLa_it_all strategy) completing map-side join procedure in one stage. Otherwise an interval tree is used as an index for additional lookup step before the equi-shuffle-join operation between s1 and s2 (SeQuiLa_it_int strategy).

Optimization algorithm

Rule-based optimizer’s algorithm chooses the most efficient join strategy.

Custom interval structure

By default SeQuiLa uses Red-Black tree with intervals implemented in org.biodatageeks.sequila.rangejoins.methods.IntervalTree.IntervalTreeRedBlack<V> class. However, it is possible to provide a custom interval structure and set it using spark.biodatageeks.rangejoin.intervalHolderClass parameter. There are 2 prerequisites:

custom class must implement interface provided by the trait: org.biodatageeks.sequila.rangejoins.methods.base.BaseIntervalHolder[V]
node class used for storing intervals must extend org.biodatageeks.sequila.rangejoins.methods.base.BaseNode[V]

Both Scala and Java classes are supported. Please use default interval tree implementation for reference.

Interval joins

Mon, 01 Jan 0001 00:00:00 +0000

Performance tests description

In order to evaluate our range join strategy we have run a number of tests using both one-node and a Hadoop cluster installations. In that way we were able to analyze both vertical (by means of adding computing resources such as CPU/RAM on one machine) as well as horizontal (by means of adding resources on multiple machines) scalability.

The main idea of the test was to compare performance of SeQuiLa with other tools like featureCounts and genAp that can be used to compute the number of reads intersecting predefined genomic intervals. It is by no means one of the most commonly used operations in both DNA and RNA-seq data processing, most notably in gene differential expression and copy number variation-calling. featureCounts performance results have been treated as a baseline. In order to show the difference between the naive approach using the default range join strategy available in Spark SQL and SeQuiLa interval-tree one, we have included it in the single-node test.

Testing enviornment

Datasets

Test name	Format	Size [GB]	Row count
WES-SN	BAM	17	161544693
WES-SN	ADAM	14	161544693
WES-SN	BED	0.0045	193557
WGS-CL	BAM	273	2617420313
WGS-CL	ADAM	215	2617420313
WGS-CL	BED	0.0016	68191

WES-SN - tests performed on a single node using WES dataset

WGS-CL - tests performed on a cluster using WGS dataset

Hadoop cluster

Masters	Workers	Hadoop distribution	Total disk HDFS [TB]	Total YARN cores	Total YARN RAM [TB]	Net [Gbits]
1	4	CDH 5.12	5	96	256	10

Apache Spark configuration

Parameter	Value	SeQuiLa only	Local-test	Cluster-test
spark.driver.memory	8g	no	yes	yes
spark.executor.memory	4g-8g	no	yes	yes
spark.executor.cores	2-4	no	yes	yes
spark.dynamicAllocation.enabled	false	no	yes	yes

Software

Software	Version
Apache Spark	2.2.1
Scala	2.11.8
JDK	1.8
ADAM	0.22
featureCounts	1.6.0

SeQuiLa

BAM table

    val bamPath = "NA12878*.bam"
    spark.sql(
      s"""
         |CREATE TABLE reads
         |USING org.biodatageeks.sequila.datasources.BAM.BAMDataSource
         |OPTIONS(path "${bamPath}")
         |
      """.stripMargin)
    spark.sql(s"SELECT contigName,start,end FROM reads LIMIT 1").show()

    +----------+-----+---+
    |contigName|start|end|
    +----------+-----+---+
    |      chr1|   34|109|
    +----------+-----+---+

ADAM table

    val adamPath = "NA12878*.adam"
    spark.sql(
      s"""
         |CREATE TABLE reads
         |USING org.biodatageeks.sequila.datasources.ADAM.ADAMDataSource
         |OPTIONS(path "${adamPath}")
         |
      """.stripMargin)
    spark.sql(s"SELECT contigName,start,end FROM reads LIMIT 1").show()

    +----------+-----+---+
    |contigName|start|end|
    +----------+-----+---+
    |      chr1|   34|109|
    +----------+-----+---+

BED table

    val  bedPath="tgp_exome_hg18.bed"
    spark.sql(s"""
        |CREATE TABLE targets
        |USING org.biodatageeks.sequila.datasources.BED.BEDDataSource
        |OPTIONS (path "file:///${bedPath}")""".stripMargin)
    spark.sql("SELECT contig,post_start, pos_end FROM targets LIMIT 1").show

    +------+---------+--------+
    |contig|pos_start| pos_end|
    +------+---------+--------+
    |  chr1|     4806|    4926|
    +------+---------+--------+

    SELECT targets.contigName,targets.start,targets.end,count(*) FROM reads JOIN targets
         ON (targets.contigName=reads.contigName
         AND
         CAST(reads.end AS INTEGER)>=CAST(targets.start AS INTEGER)
         AND
         CAST(reads.start AS INTEGER)<=CAST(targets.end AS INTEGER)
         )
         GROUP BY targets.contigName,targets.start,targets.end

Results single node

Results Hadoop cluster

Conclusions

SeQuiLa when run in parallel outperforms selected competing tools in terms of speed on single node (1.7-22.1x) and cluster (3.2-4.7x). SeQuiLa strategy involving broadcasting interval forest with all data columns (SeQuiLa_it_all) performs best in most of the cases (no network shuffling required), whereas broadcasting intervals with identifiers only (SeQuiLa_it_int) performs comparable to, or better than GenAp. All algorithms favours columnar (ADAM) to row oriented (BAM) file format due to column pruning and disk I/O operations reduction.

Interval joins

Mon, 01 Jan 0001 00:00:00 +0000

Property Name	Default	Meaning
spark.biodatageeks.rangejoin.useJoinOrder	false	Whether to always broadcast the right table of a join or to computer row counts and pick up the smaller one.
spark.biodatageeks.rangejoin.maxBroadcastSize	0.1*spark.driver.memory	The maximum allowed size of the broadcasted intverval structure which is used by the SeQuiLa’s optimizer to chose interval join algorithm.
spark.biodatageeks.rangejoin.maxGap	0	The maximum gap between between regions
spark.biodatageeks.rangejoin.minOverlap	1	The minimal length of the overlap between regions
spark.biodatageeks.rangejoin.intervalHolderClass	`IntervalTreeRedBlack`	Pluggable mechanism for implementing custom interval structures.

VCF

Mon, 01 Jan 0001 00:00:00 +0000

TBD

SeQuiLa-cov: A fast and scalable library for depth of coverage calculations

Thu, 01 Aug 2019 00:00:00 +0000

Click the Cite button above to demo the feature to enable visitors to import publication metadata into their reference management software.

Supplementary notes can be added here, including code and math.

SeQuiLa: an elastic, fast and scalable SQL-oriented solution for processing and querying genomic intervals

Sat, 01 Jun 2019 00:00:00 +0000

Click the Cite button above to demo the feature to enable visitors to import publication metadata into their reference management software.

Supplementary notes can be added here, including code and math.

Search Results

Mon, 01 Jan 0001 00:00:00 +0000

SeQuiLa | sequila

BAM

Examples

CRAM

Examples

General

Pileup and coverage

Pileup and coverage

Testing environment

Datasets

Tools

Single node specification

Hadoop cluster

SeQuiLa, ADAM and GATK(Spark) parameters

Other parameters

SeQuiLa

Coverage

Pileup

Results single node

Coverage

Pileup

Results Hadoop cluster

Coverage

Pileup

Python

Table of Contents

Watchout

sequila

SequilaSession Objects

Builder Objects

getOrCreate

Examples

coverage

Parameters

Returns

Examples

pileup

Parameters

Returns

Examples

SQL

Table of Contents

Watchout

coverage

Parameters

Examples

coverage

Parameters

Examples

Pileup and coverage

BED

Examples

Interval joins

Overview

Tables structure

Algorithm

Interval tree

Broadcasting mechanism

Optimizations

Optimization algorithm

Custom interval structure

Interval joins

Performance tests description

Testing enviornment

Datasets

Hadoop cluster

Apache Spark configuration

Software

SeQuiLa

BAM table

ADAM table

BED table

Results single node

Results Hadoop cluster

Conclusions

Interval joins

VCF

SeQuiLa-cov: A fast and scalable library for depth of coverage calculations

SeQuiLa: an elastic, fast and scalable SQL-oriented solution for processing and querying genomic intervals

Search Results