Last active
May 19, 2020 14:46
-
-
Save cjw85/8e2a647cebbc719ba53bb023efcf9a5c to your computer and use it in GitHub Desktop.
A minimal fasta/q parser in python using kseq.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* The MIT License | |
Permission is hereby granted, free of charge, to any person obtaining | |
a copy of this software and associated documentation files (the | |
"Software"), to deal in the Software without restriction, including | |
without limitation the rights to use, copy, modify, merge, publish, | |
distribute, sublicense, and/or sell copies of the Software, and to | |
permit persons to whom the Software is furnished to do so, subject to | |
the following conditions: | |
The above copyright notice and this permission notice shall be | |
included in all copies or substantial portions of the Software. | |
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
SOFTWARE. | |
*/ | |
from cffi import FFI | |
def build(): | |
ffibuilder = FFI() | |
ffibuilder.set_source( | |
"_fastx", | |
r""" | |
#include <zlib.h> | |
#include <stdio.h> | |
#include "kseq.h" | |
KSEQ_INIT(gzFile, gzread) | |
kseq_t *open_fastx (const char *fn) { | |
gzFile fp = gzopen(fn, "r"); | |
return kseq_init(fp); | |
} | |
void close_fastx(kseq_t *kseq) { | |
gzclose(kseq->f->f); | |
kseq_destroy(kseq); | |
} | |
""", | |
libraries=['z'] | |
) | |
ffibuilder.cdef( | |
r""" | |
typedef struct kstring_t { | |
size_t l, m; char *s; | |
} kstring_t; | |
typedef struct kseq_t { | |
kstring_t name, comment, seq, qual; | |
...; | |
} kseq_t; | |
kseq_t *open_fastx(const char *fn); | |
void close_fastx(kseq_t *kseq); | |
int kseq_read(kseq_t *kseq); | |
""" | |
) | |
ffibuilder.compile(verbose=True) | |
def run(): | |
import _fastx | |
import sys | |
class Fastx: | |
def __init__(self, fname): | |
self.kseq = _fastx.lib.open_fastx(fname.encode()) | |
def __del__(self): | |
_fastx.lib.close_fastx(self.kseq) | |
def __next__(self): | |
if _fastx.lib.kseq_read(self.kseq) >= 0: | |
return ( | |
_fastx.ffi.string(self.kseq.name.s), | |
_fastx.ffi.string(self.kseq.seq.s), | |
_fastx.ffi.string(self.kseq.qual.s) if self.kseq.qual.l > 0 else "") | |
else: | |
raise StopIteration() | |
def __iter__(self): | |
return self | |
n, slen, qlen = 0, 0, 0 | |
for name, seq, qual in Fastx(sys.argv[1]): | |
n += 1 | |
slen += len(seq) | |
qlen += qual and len(qual) or 0 | |
print('{}\t{}\t{}'.format(n, slen, qlen)) | |
if __name__ == "__main__": | |
import sys | |
if len(sys.argv) < 2: | |
build() | |
else: | |
run() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This was created because I found the results for fastq parsing with pyfastx presented here to be a little odd: since pyfastx uses
kseq.h
one would expect near the baseline C speed (which also useskseq.h
).If we line profile the code then the original program spends as much time adding python objects together as it does in the pyfastx package:
The code in this gist works under both CPython and PyPy, unlike pyfastx which is strictly a CPython extension. When using PyPy, the difference between the Python and C implementation is narrowed dramatically: