Commit ffe8e964 authored by Jenda's avatar Jenda

scanner

parent df1d2083
......@@ -22,6 +22,11 @@ def getfir(samplerate, filtertype, bw, transition, maxtaps):
"""
coefs = None
if filtertype not in ["hamming", "rcos"]:
print("Unknown filter type %s, defaulting to hamming"%xl.filtertype)
filtertype = "hamming"
if filtertype == "hamming":
while True:
coefs = firdes.low_pass(1, samplerate, bw, transition, firdes.WIN_HAMMING)
......@@ -36,8 +41,6 @@ def getfir(samplerate, filtertype, bw, transition, maxtaps):
print("The requested raised cosine filter of len %i is too long. Limiting to %i."%(transition, maxtaps))
transition = maxtaps
coefs = firdes.root_raised_cosine(1, samplerate, bw, 0.35, transition)
else:
showerror("Unknown filter type %s"%xl.filtertype)
return coefs
......@@ -24,7 +24,7 @@ def channelhelper():
COMPLEX64 = 8
class scanner():
class KukurukuScanner():
def __init__(self, l, confdir):
self.l = l
......@@ -35,7 +35,7 @@ class scanner():
return True
if frag[:2] == "*/":
num = safe_cast(frag[2:], int, None)
if not num:
if not num or num == 0:
self.l.l("Invalid modulo number %s"%frag[2:], "CRIT")
return False
if i % num == 0:
......@@ -67,7 +67,16 @@ class scanner():
def getfn(self, f, rate):
ds = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d-%H-%M-%S')
return "%i-%s-%i"%(f, ds, rate)
if rate is not None:
return "%i-%s-%i"%(f, ds, rate)
else:
return "%i-%s"%(f, ds)
def dump_spectrum(self, acc, filename):
f = open(filename, "wb")
for flt in acc:
f.write("%f\n"%flt)
f.close()
def record_long(self, cronframe):
''' Record selected channels from a scheduled scan '''
......@@ -79,14 +88,15 @@ class scanner():
for channel in cronframe.channels:
peaks.append((channel.freq-cronframe.freq, channel.bw, channel))
self.do_record(peaks, cronframe.cronlen, cronframe.stickactivity, 1, channel.freq, cronframe.floor, None)
self.do_record(peaks, cronframe.cronlen, cronframe.stickactivity, 1, None, cronframe)
def scan(self, scanframe):
''' Find peaks in spectrum, record if specified in allow/blacklist '''
starttime = time.time()
self.sdr.tune(scanframe.freq)
self.sdr.set_gain(self.conf.messgain, scanframe.gain)
self.sdrflush()
self.l.l("scan: tune %ik"%(scanframe.freq/1000), "INFO")
self.l.l("scan: tune %ik, gain %i"%(scanframe.freq/1000, scanframe.gain), "INFO")
delta = self.conf.interval - time.time() % self.conf.interval
nbytes = int(self.conf.rate * COMPLEX64 * delta)
......@@ -95,27 +105,40 @@ class scanner():
acc = self.compute_spectrum(sbuf)
if self.conf.dumpspectrum == "always":
self.dump_spectrum(acc, self.getfn(scanframe.freq, None)+".spectrum.txt")
floor = sorted(acc)[int(scanframe.floor * self.conf.fftw)]
peaks = self.find_peaks(acc, floor + scanframe.sql)
peaks = self.filter_blacklist(peaks, scanframe.freq)
self.do_record(peaks, scanframe.stick, scanframe.stickactivity, self.conf.filtermargin, scanframe.freq, scanframe.floor, sbuf)
if len(peaks) == 0:
histo = self.compute_histogram(sbuf[:COMPLEX64*self.conf.fftw])
self.update_and_set_gain(scanframe, histo)
self.sdr.set_gain(self.conf.messgain, scanframe.gain)
else:
if self.conf.dumpspectrum == "on_signal":
self.dump_spectrum(acc, self.getfn(scanframe.freq, None)+".spectrum.txt")
self.do_record(peaks, scanframe.stick, self.conf.filtermargin, sbuf, scanframe)
def filter_blacklist(self, peaks, center):
ret = []
if not self.conf.blacklist:
return peaks
for peak in peaks:
f = peak[0]+center-peak[1]/2
print(self.conf.blacklist)
pos = bisect.bisect(self.conf.blacklist, f)
f = peak[0]+center
pos = bisect.bisect(self.conf.blacklist, (f, None))
if pos >= len(self.conf.blacklist):
ret.append(peak)
continue
pos -= 1
entry = self.conf.blacklist[pos]
if entry-f > peak[1]:
if f < entry[0] or f > entry[1]:
ret.append(peak)
continue
......@@ -123,9 +146,23 @@ class scanner():
return ret
def do_record(self, peaks, stoptime, stickactivity, safetymargin, center, floor, buf):
def update_and_set_gain(self, frame, histo):
diff = 0
# if there is some signal in the highest 10 bins, decrease gain
if sum(histo[-10:]) > 0:
diff = -1
# if there is no signal in the upper half, increase gain
if sum(histo[-50:]) == 0:
diff = 1
frame.gain += diff
frame.gain = np.clip(frame.gain, self.conf.mingain, self.conf.maxgain)
def do_record(self, peaks, stoptime, safetymargin, buf, frame):
lastact = time.time()
center = frame.freq
floor = frame.floor
helpers = []
for peak in peaks:
......@@ -145,21 +182,31 @@ class scanner():
ch.cylen = len(ch.taps)
ch.carry = '\0' * ch.cylen
basename = self.getfn(f+center, ch.rate)
if len(peak) >= 3 and peak[2].pipe is not None:
(ch.fd_r, ch.file) = os.pipe()
subprocess.Popen([peak[2].pipe], shell=True, stdin=ch.fd_r, bufsize=-1)
self.l.l("Recording %i (PIPE)"%f, "INFO")
cmdline = peak[2].pipe.replace("_FILENAME_", basename)
subprocess.Popen([cmdline], shell=True, stdin=ch.fd_r, bufsize=-1)
self.l.l("Recording \"%s\" (PIPE), firlen %i"%(cmdline, len(taps)), "INFO")
else:
ch.file = os.open(self.getfn(f+center, ch.rate) + ".cfile", os.O_WRONLY|os.O_CREAT)
self.l.l("Recording %s"%ch.file, "INFO")
fullfile = basename + ".cfile"
ch.file = os.open(fullfile, os.O_WRONLY|os.O_CREAT)
ch.fd_r = None
self.l.l("Recording to file \"%s\", firlen %i"%(fullfile, len(taps)), "INFO")
helpers.append(ch)
while True:
while True and len(helpers) > 0:
# read data from sdr if needed
if buf is None:
# read data from sdr
buf = self.pipefile.read(self.conf.bufsize * COMPLEX64)
if frame.stickactivity:
acc = self.compute_spectrum(buf)
for peak in peaks:
if self.check_activity(acc, peak, floor):
lastact = time.time()
self.l.l("%f has activity, continuing"%peak[0], "INFO")
# xlate each channel
for ch in helpers:
......@@ -168,17 +215,14 @@ class scanner():
int(ch.decim), ch.rotator, ch.rotpos, ch.firpos, ch.file)
ch.carry = buf[-ch.cylen:]
if stickactivity:
acc = self.compute_spectrum(buf)
for peak in peaks:
if self.check_activity(acc, peak, floor):
lastact = time.time()
self.l.l("%f has activity, continuing"%peak[0], "INFO")
if time.time() > lastact + stoptime:
self.l.l("Record stop", "INFO")
break
histo = self.compute_histogram(buf[:COMPLEX64*self.conf.fftw])
self.update_and_set_gain(frame, histo)
self.sdr.set_gain(self.conf.messgain, frame.gain)
buf = None
for ch in helpers:
......@@ -200,6 +244,15 @@ class scanner():
return False
def compute_histogram(self, sbuf):
""" Compute histogram with 100 bins """
acc = np.zeros(100)
dt = np.dtype("=f4")
buf = np.frombuffer(sbuf, dtype=dt)
for i in range(0, len(buf)):
acc[np.clip(int(buf[i]*99), 0, 99)] += 1
return acc
def compute_spectrum(self, sbuf):
acc = np.zeros(self.conf.fftw)
......
......@@ -3,7 +3,7 @@ SWIG = swig3.0
CFLAGS = -Wall -lm -lvolk -std=gnu99 -O3 -g -ggdb3
all: xlater.py
all: _xlater.so
.PHONY: clean
......@@ -11,10 +11,10 @@ xlater_wrap.c: xlater.c
$(SWIG) -python xlater.i
xlater.o: xlater_wrap.c
$(CC) $(CFLAGS) -c -fpic xlater.c xlater_wrap.c -I/usr/include/python2.7
$(CC) -c -fpic xlater.c xlater_wrap.c -I/usr/include/python2.7 $(CFLAGS)
xlater.py: xlater.o
$(CC) $(CFLAGS) -shared xlater.o xlater_wrap.o -o _xlater.so
_xlater.so: xlater.o
$(CC) -shared xlater.o xlater_wrap.o -o _xlater.so $(CFLAGS)
clean:
......
99700000 64000 bonton
h 99700000 64000 bonton
......@@ -5,16 +5,16 @@ sql=0.3
cron=* 23 * * *
cronlen=60
randscan=no
stickactivity=True
stickactivity=false
[Channel1]
freq=99700
bw=64
continue=30
;continue=30
;pipe=/home/jenda/tmp/kukuruku/client/modes/wfm.py -r 0.44
[Channel2]
freq=99300
bw=64
continue=30
;continue=30
;pipe=/home/jenda/tmp/kukuruku/client/modes/wfm.py -r 0.44
[General]
; size of SDR buffer, used for flushing
bufsize=2048000
bufsize=1000000
; SDR sample rate
rate=2048000
rate=10000000
; the duration of one tune, in seconds
interval=10
interval=5
; do not bother with scan if we have less than this number of seconds left
skip=5
skip=3
; spectrum transform size
fftw=2048
fftw=16384
; transform decimation
; we do transforms on every fftw*fftskip offset
......@@ -30,16 +30,16 @@ overlap=0.2
nonce="abcdef"
; the noise floor is set as floor-quantile from power spectrum
floor=0.3
floor=0.4
; treat everything higher than floor+sql as a signal
sql=0.3
sql=0.5
; do not care about signals narrower than this (kHz) (e.g. random interference, carrier-only etc.)
minw=10
; do not care about signals wider than this (kHz) (e.g. do not catch 8MHz wide DVB-T multiplex)
maxw=150
; do not care about signals wider than this (kHz) (e.g. do not catch 8 MHz wide DVB-T multiplex)
maxw=200
; initial gain
gain=0,30,30,30
......@@ -57,7 +57,7 @@ messgain=1
mingain=10
maxgain=49
; when generating FIR filters, make transition band this wide
; when generating FIR filters, make transition band bandwidth_of_signal*transition wide
transition=0.2
; multiply the filnal filter width by this factor
......@@ -67,3 +67,8 @@ filtermargin = 1.5
stickactivity = False
stick = 10
; never
; on_signal
; always
dumpspectrum = always
......@@ -18,24 +18,18 @@ import tempfile
class top_block(gr.top_block):
def __init__(self, device, rate, freq, gain, fd):
def __init__(self, device, rate, ppm, fd):
self.rate = rate
self.gain = gain
gr.top_block.__init__(self, "Top Block")
self.osmosdr_source = osmosdr.source(device)
self.osmosdr_source.set_sample_rate(int(rate))
self.osmosdr_source.set_center_freq(int(freq), 0)
self.osmosdr_source.set_freq_corr(0, 0)
self.osmosdr_source.set_freq_corr(ppm, 0)
self.osmosdr_source.set_dc_offset_mode(0, 0)
self.osmosdr_source.set_iq_balance_mode(2, 0)
self.osmosdr_source.set_gain_mode(False, 0)
self.osmosdr_source.set_gain(int(gain), 0)
self.osmosdr_source.set_if_gain(int(gain), 0)
self.osmosdr_source.set_bb_gain(int(gain), 0)
self.osmosdr_source.set_antenna("", 0)
self.osmosdr_source.set_bandwidth(0, 0)
#self.osmosdr_source.set_bandwidth(0, 0)
self.blocks_file_descriptor_sink_0 = blocks.file_descriptor_sink(gr.sizeof_float*2, fd)
......@@ -48,7 +42,19 @@ class top_block(gr.top_block):
def tune(self, f):
self.osmosdr_source.set_center_freq(int(f), 0)
import KukurukuScanner
def set_gain(self, pos, val):
if pos == 0:
self.osmosdr_source.set_gain_mode((val != 0), 0)
elif pos == 1:
self.osmosdr_source.set_gain(val, 0)
elif pos == 2:
self.osmosdr_source.set_if_gain(val, 0)
elif pos == 3:
self.osmosdr_source.set_bb_gain(val, 0)
else:
raise Exceprion("Invalid gain number %s"%val)
from KukurukuScanner import KukurukuScanner
import util
def usage():
......@@ -60,15 +66,15 @@ device = ""
confdir = None
try:
(opts, args) = getopt.getopt(sys.argv[1:], "d:r:p:c:")
(opts, args) = getopt.getopt(sys.argv[1:], "d:c:p:")
except getopt.GetoptError as e:
usage()
ppm = 0
device = ""
for opt, arg in opts:
if opt == "-d":
device = arg
elif opt == "-r":
rate = arg
elif opt == "-c":
confdir = arg
elif opt == "-p":
......@@ -85,10 +91,12 @@ l.setloglevel("DBG")
(fd_r,fd_w) = os.pipe()
scanner = KukurukuScanner.scanner(l, confdir)
scanner = KukurukuScanner(l, confdir)
sdr = top_block(device, scanner.conf.rate, 0, 10, fd_w)
sdr = top_block(device, scanner.conf.rate, ppm, fd_w)
for i in range(0, 4):
sdr.set_gain(i, int(scanner.conf.gainpcs[i]))
file_r = os.fdopen(fd_r, "rb")
......
......@@ -18,7 +18,10 @@ archdir = "archive"
if not os.path.isdir(archdir):
os.mkdir(archdir)
filenames = os.listdir(".")
if len(sys.argv) > 1:
filenames = sys.argv[1:]
else:
filenames = os.listdir(".")
for filename in filenames:
if filename[-6:] != ".cfile":
......@@ -31,6 +34,7 @@ for filename in filenames:
p2 = p[7].split(".")
rate = float(p2[0])
freq = int(p[0])
while True:
print("Filename: %s | bandwidth %ik"%(filename, rate/1000))
......@@ -41,7 +45,8 @@ for filename in filenames:
print("_n) ignore once")
print("_s) ignore by sorter")
print("_h) ignore by scanner")
c = raw_input('Choice: __ [optional comment]')
print("q) just quit")
c = raw_input('Choice: __ [optional comment] ')
if c.isdigit():
modename = modes.keys()[int(c)]
......@@ -57,19 +62,28 @@ for filename in filenames:
print("resample ratio = %f"%resample)
program += " -r %f"%resample
process = subprocess.Popen(["cat " + filename + " | " + program], shell=True)
process.wait()
elif c[0] == "q":
sys.exit(0)
elif len(c) >= 2:
if c[0] == "d":
os.remove(filename)
elif c[0] == "a":
os.remane(filename, archdir + "/" + filename)
os.rename(filename, archdir + "/" + filename)
if c[1] == "s":
pass
bl = open("blacklist.txt", "a")
bl.write("s %i %i %s\n"%(freq, rate, c[3:]))
bl.close()
elif c[1] == "h":
pass
bl = open("blacklist.txt", "a")
bl.write("h %i %i %s\n"%(freq, rate, c[3:]))
bl.close()
break
else:
print("unknown input %s"%c)
......@@ -4,6 +4,7 @@ from __future__ import print_function
import configparser
import os
import sys
from framespec import scanframe, cronframe, channel
import math
from threading import Lock
......@@ -40,13 +41,24 @@ class ConfReader():
self.gain = rc.get(MAINSECTION, 'gain')
self.stickactivity = rc.getboolean(MAINSECTION, 'stickactivity')
self.stick = rc.getint(MAINSECTION, 'stick')
self.dumpspectrum = rc.get(MAINSECTION, 'dumpspectrum')
self.gainpcs = self.gain.split(",")
if len(self.gainpcs) != 4:
print("Wrong gain string format in configuration %s, should be \"N,N,N,N\""%self.gain)
sys.exit(1)
if self.messgain < 0 or self.messgain > 3:
print("messagin parameter must be an integer from range [0, 3]")
sys.exit(1)
defaultgain = int(self.gainpcs[self.messgain])
files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
self.scanframes = []
self.cronframes = []
step = float(self.rate) * (1-self.overlap)
step = float(self.rate) * (1.0-self.overlap)
# Read scanframe and channel config files
for f in files:
......@@ -84,16 +96,20 @@ class ConfReader():
freqstart = rc.getint(MAINSECTION, "freqstart")*1000 + step/2
freqstop = rc.getint(MAINSECTION, "freqstop")*1000 - step/2
numf = int(math.ceil(float(freqstop - freqstart)/step))
delta = float(freqstop - freqstart) / numf
if numf == 0: # freqstart == freqstop
numf = 1
delta = freqstop
else:
delta = float(freqstop - freqstart) / numf
cf = freqstart
while cf < math.ceil(freqstop):
while cf <= math.ceil(freqstop):
frm = scanframe()
frm.freq = cf
frm.floor = floor
frm.stickactivity = stickactivity
frm.stick = stick
frm.sql = sql
frm.gain = self.gain
frm.gain = defaultgain
self.scanframes.append(frm)
print("insert %f"%cf)
......@@ -107,7 +123,7 @@ class ConfReader():
frm.stickactivity = stickactivity
frm.stick = stick
frm.sql = sql
frm.gain = self.gain
frm.gain = defaultgain
frm.cron = rc.get(MAINSECTION, "cron")
frm.cronlen = rc.getint(MAINSECTION, "cronlen")
frm.channels = channels
......@@ -127,7 +143,7 @@ class ConfReader():
frm.stickactivity = stickactivity
frm.stick = stick
frm.sql = sql
frm.gain = self.gain
frm.gain = defaultgain
channels = self.channel_list_from_config(rc)
frm.channels = channels
......@@ -135,16 +151,30 @@ class ConfReader():
self.scanframes.append(frm)
# read blacklist
self.blacklist = []
bl = []
if os.path.isfile("blacklist.txt"):
f = open("blacklist.txt")
lines = f.readlines()
for line in lines:
pieces = line.strip().split(" ")
if len(pieces) >= 2:
#self.blacklist.append((int(pieces[0]), int(pieces[1])))
self.blacklist.append(int(pieces[0]))
self.blacklist.sort()
if len(pieces) >= 3:
if pieces[0] == "h":
freq = int(pieces[1])
bw = int(pieces[2])/2
bl.append((freq-bw, freq+bw))
self.blacklist = []
if len(bl) > 0:
# sort and union intervals
bl.sort(key=lambda x: x[0])
self.blacklist.append(bl[0])
for interval in bl[1:]:
if self.blacklist[-1][1] < interval[0]:
self.blacklist.append(interval)
elif self.blacklist[-1][1] == interval[0]:
self.blacklist[-1][1] = interval[1]
print(self.blacklist)
def channel_list_from_config(self, rc):
......
......@@ -54,6 +54,8 @@ int xdump(char * _buf, size_t buflen, char * _carry, size_t carrylen, char * _ta
*firpos = i - nsamples;
free(alldata);
//fclose(of);
return 0;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment