Commit 06f10981 authored by Jenda's avatar Jenda
Browse files

scanner

parent 4377bbef
...@@ -26,9 +26,9 @@ COMPLEX64 = 8 ...@@ -26,9 +26,9 @@ COMPLEX64 = 8
class KukurukuScanner(): class KukurukuScanner():
def __init__(self, l, confdir): def __init__(self, l):
self.l = l self.l = l
self.conf = util.ConfReader(confdir) self.conf = util.ConfReader()
def croncmp(self, i, frag): def croncmp(self, i, frag):
if frag == "*": if frag == "*":
...@@ -86,9 +86,9 @@ class KukurukuScanner(): ...@@ -86,9 +86,9 @@ class KukurukuScanner():
self.sdrflush() self.sdrflush()
peaks = [] peaks = []
for channel in cronframe.channels: for channel in cronframe.channels:
peaks.append((channel.freq-cronframe.freq, channel.bw, channel)) peaks.append([channel.freq-cronframe.freq, channel.bw, channel])
self.do_record(peaks, cronframe.cronlen, 1, None, cronframe) self.do_record(peaks, 1, None, cronframe)
def scan(self, scanframe): def scan(self, scanframe):
''' Find peaks in spectrum, record if specified in allow/blacklist ''' ''' Find peaks in spectrum, record if specified in allow/blacklist '''
...@@ -121,7 +121,25 @@ class KukurukuScanner(): ...@@ -121,7 +121,25 @@ class KukurukuScanner():
else: else:
if self.conf.dumpspectrum == "on_signal": if self.conf.dumpspectrum == "on_signal":
self.dump_spectrum(acc, self.getfn(scanframe.freq, None)+".spectrum.txt") self.dump_spectrum(acc, self.getfn(scanframe.freq, None)+".spectrum.txt")
self.do_record(peaks, scanframe.stick, self.conf.filtermargin, sbuf, scanframe)
# determine whether we stumbled upon any specified channel which has PIPE set
peaks2 = []
for peak in peaks:
modified = False
for channel in scanframe.channels:
if channel.freq - scanframe.freq - channel.bw/2 < peak[0] and \
channel.freq - scanframe.freq + channel.bw/2 > peak[0]:
if channel.pipe:
peaks2.append([channel.freq - scanframe.freq, channel.bw, channel.pipe])
else:
peaks2.append([channel.freq - scanframe.freq, channel.bw])
modified = True
if not modified:
peaks2.append(peak)
self.do_record(peaks2, self.conf.filtermargin, sbuf, scanframe)
def filter_blacklist(self, peaks, center): def filter_blacklist(self, peaks, center):
ret = [] ret = []
...@@ -158,11 +176,12 @@ class KukurukuScanner(): ...@@ -158,11 +176,12 @@ class KukurukuScanner():
frame.gain += diff frame.gain += diff
frame.gain = np.clip(frame.gain, self.conf.mingain, self.conf.maxgain) frame.gain = np.clip(frame.gain, self.conf.mingain, self.conf.maxgain)
def do_record(self, peaks, stoptime, safetymargin, buf, frame): def do_record(self, peaks, safetymargin, buf, frame):
lastact = time.time() lastact = time.time()
center = frame.freq center = frame.freq
floor = frame.floor floor = frame.floor
stoptime = time.time() + frame.stick
helpers = [] helpers = []
for peak in peaks: for peak in peaks:
...@@ -183,9 +202,9 @@ class KukurukuScanner(): ...@@ -183,9 +202,9 @@ class KukurukuScanner():
ch.carry = '\0' * ch.cylen ch.carry = '\0' * ch.cylen
basename = self.getfn(f+center, ch.rate) basename = self.getfn(f+center, ch.rate)
if len(peak) >= 3 and peak[2].pipe is not None: if len(peak) >= 3 and peak[2] is not None:
(ch.fd_r, ch.file) = os.pipe() (ch.fd_r, ch.file) = os.pipe()
cmdline = peak[2].pipe.replace("_FILENAME_", basename) cmdline = peak[2].replace("_FILENAME_", basename)
subprocess.Popen([cmdline], shell=True, stdin=ch.fd_r, bufsize=-1) subprocess.Popen([cmdline], shell=True, stdin=ch.fd_r, bufsize=-1)
self.l.l("Recording \"%s\" (PIPE), firlen %i"%(cmdline, len(taps)), "INFO") self.l.l("Recording \"%s\" (PIPE), firlen %i"%(cmdline, len(taps)), "INFO")
else: else:
...@@ -201,6 +220,7 @@ class KukurukuScanner(): ...@@ -201,6 +220,7 @@ class KukurukuScanner():
# read data from sdr if needed # read data from sdr if needed
if buf is None: if buf is None:
buf = self.pipefile.read(self.conf.bufsize * COMPLEX64) buf = self.pipefile.read(self.conf.bufsize * COMPLEX64)
if frame.stickactivity: if frame.stickactivity:
acc = self.compute_spectrum(buf) acc = self.compute_spectrum(buf)
for peak in peaks: for peak in peaks:
...@@ -215,7 +235,8 @@ class KukurukuScanner(): ...@@ -215,7 +235,8 @@ class KukurukuScanner():
int(ch.decim), ch.rotator, ch.rotpos, ch.firpos, ch.file) int(ch.decim), ch.rotator, ch.rotpos, ch.firpos, ch.file)
ch.carry = buf[-ch.cylen:] ch.carry = buf[-ch.cylen:]
if time.time() > lastact + stoptime: if ((not frame.stickactivity) and time.time() > stoptime) or \
(frame.stickactivity and time.time() > lastact + frame.silencegap):
self.l.l("Record stop", "INFO") self.l.l("Record stop", "INFO")
break break
...@@ -235,11 +256,14 @@ class KukurukuScanner(): ...@@ -235,11 +256,14 @@ class KukurukuScanner():
binhz = self.conf.rate/self.conf.fftw binhz = self.conf.rate/self.conf.fftw
startbin = int(peak[0]/binhz - peak[1]/(2*binhz)) startbin = int(peak[0]/binhz - peak[1]/(2*binhz)) + self.conf.fftw/2
stopbin = int(peak[0]/binhz + peak[1]/(2*binhz)) stopbin = int(peak[0]/binhz + peak[1]/(2*binhz)) + self.conf.fftw/2
print(acc)
for i in range(startbin, stopbin): for i in range(startbin, stopbin):
if acc[i] > floor: if acc[i] > floor:
print("acc %i lvl %f floor %f"%(i,acc[i], floor))
return True return True
return False return False
...@@ -263,13 +287,14 @@ class KukurukuScanner(): ...@@ -263,13 +287,14 @@ class KukurukuScanner():
buf = buf*self.window buf = buf*self.window
fft = np.absolute(np.fft.fft(buf)) fft = np.fft.fft(buf)
fft = (np.real(fft) * np.real(fft) + np.imag(fft) * np.imag(fft))/self.conf.fftw
fft = np.log10(fft)*10
acc += fft acc += fft
iters += 1 iters += 1
acc = np.divide(acc, iters) acc = np.divide(acc, iters)
acc = np.log(acc)
# FFT yields a list of positive frequencies and then negative frequencies. # FFT yields a list of positive frequencies and then negative frequencies.
# We want it in the "natural" order. # We want it in the "natural" order.
...@@ -302,7 +327,7 @@ class KukurukuScanner(): ...@@ -302,7 +327,7 @@ class KukurukuScanner():
if (i-first) >= minspan and (i-first) <= maxspan: if (i-first) >= minspan and (i-first) <= maxspan:
f = binhz*(((i+first)/2)-self.conf.fftw/2) f = binhz*(((i+first)/2)-self.conf.fftw/2)
w = binhz*(i-first) w = binhz*(i-first)
peaks.append((f, w)) peaks.append([f, w])
self.l.l("signal at %f width %f"%(f, w), "INFO") self.l.l("signal at %f width %f"%(f, w), "INFO")
return peaks return peaks
......
h 99700000 64000 bonton
[General]
freq=100000
floor=0.2
sql=0.3
cron=* 16 * * *
cronlen=60
randscan=no
stickactivity=false
[Channel1]
freq=99700
bw=64
;continue=30
;pipe=/home/jenda/tmp/kukuruku/client/modes/wfm.py -r 0.44
[Channel2]
freq=99300
bw=64
;continue=30
;pipe=/home/jenda/tmp/kukuruku/client/modes/wfm.py -r 0.44
[General]
; size of SDR buffer, used for flushing
bufsize=2048000
; SDR sample rate
rate=2048000
; the duration of one tune, in seconds
interval=5
; do not bother with scan if we have less than this number of seconds left
skip=3
; spectrum transform size
fftw=2048
; transform decimation
; we do transforms on every fftw*fftskip offset
fftskip=32
; when generating scanplan of frequency range, create at least this overlap between tunes
; e.g. if we are scanning from 100 MHz with 1MHz wire SDR, the first scan will be 99.8-100.8,
; the second 100.6 - 101.6 etc.
overlap=0.2
; when the scanner picks the next frequency, it computes SHA(time+nonce)
; set this to different values if you want multiple scanners jumping independently
nonce="abcdef"
; the noise floor is set as floor-quantile from power spectrum
floor=0.4
; treat everything higher than floor+sql as a signal
sql=0.5
; do not care about signals narrower than this (kHz) (e.g. random interference, carrier-only etc.)
minw=10
; do not care about signals wider than this (kHz) (e.g. do not catch 8 MHz wide DVB-T multiplex)
maxw=200
; initial gain
gain=0,30,30,30
; try to do autogain with this gain ID parameter
; -1 .. autogain disabled
; 0 .. libosmosdr autogain
; 1 .. libosmosdr RF gain
; 2 .. libosmosdr IF gain
; 3 .. libosmosdr BB gain
; all of them are hardware-specific, if you don't know, you probably want RF gain
messgain=1
; limit the auto gain algorithm -- it should not set gain below and above this
mingain=10
maxgain=49
; when generating FIR filters, make transition band bandwidth_of_signal*transition wide
transition=0.2
; multiply the filnal filter width by this factor
; we usually don't get the exact position of the signal, so we better make the
; filter wider and hope we dump it all...
filtermargin = 1.5
stickactivity = False
stick = 10
; never
; on_signal
; always
dumpspectrum = always
[General]
freq=172500
floor=0.2
sql=1.0
cron=* 2 * * *
cronlen=60
randscan=no
[Channel1]
freq=172650
bw=16
continue=30
[Channel2]
freq=172950
bw=16
continue=30
from libutil import Struct from libutil import Struct
def scanframe(): def scanframe():
ScanframeT = Struct("scanframe", "freq floor squelch channels gain stickactivity pipe") ScanframeT = Struct("scanframe", "")
return ScanframeT(None, None, None, [], 0, False, None) return ScanframeT()
def cronframe(): def cronframe():
CronframeT = Struct("cronframe", "freq floor squelch cronstr cronlen channels gain stickactivity pipe") CronframeT = Struct("cronframe", "")
return CronframeT(None, None, None, "", 0, [], 0, False, None) return CronframeT()
def channel(): def channel():
ChannelT = Struct("channel", "freq bw cont") ChannelT = Struct("channel", "")
return ChannelT(None, None, 0) return ChannelT()
...@@ -35,10 +35,6 @@ class top_block(gr.top_block): ...@@ -35,10 +35,6 @@ class top_block(gr.top_block):
self.connect((self.osmosdr_source, 0), (self.blocks_file_descriptor_sink_0, 0)) self.connect((self.osmosdr_source, 0), (self.blocks_file_descriptor_sink_0, 0))
# osmosdr.source.get_sample_rate() seems to be broken
def get_sample_rate(self):
return self.rate
def tune(self, f): def tune(self, f):
self.osmosdr_source.set_center_freq(int(f), 0) self.osmosdr_source.set_center_freq(int(f), 0)
...@@ -58,15 +54,14 @@ from KukurukuScanner import KukurukuScanner ...@@ -58,15 +54,14 @@ from KukurukuScanner import KukurukuScanner
import util import util
def usage(): def usage():
print("Usage: %s [-d device] [-p ppm] [-r rate] -c confdir"%sys.argv[0]) print("Usage: %s [-d device] [-p ppm]"%sys.argv[0])
sys.exit(1) sys.exit(1)
ppm = 0 ppm = 0
device = "" device = ""
confdir = None
try: try:
(opts, args) = getopt.getopt(sys.argv[1:], "d:c:p:") (opts, args) = getopt.getopt(sys.argv[1:], "d:p:")
except getopt.GetoptError as e: except getopt.GetoptError as e:
usage() usage()
...@@ -75,23 +70,18 @@ device = "" ...@@ -75,23 +70,18 @@ device = ""
for opt, arg in opts: for opt, arg in opts:
if opt == "-d": if opt == "-d":
device = arg device = arg
elif opt == "-c":
confdir = arg
elif opt == "-p": elif opt == "-p":
ppm = arg ppm = arg
else: else:
usage() usage()
assert False assert False
if not confdir:
usage()
l = util.logger() l = util.logger()
l.setloglevel("DBG") l.setloglevel("DBG")
(fd_r,fd_w) = os.pipe() (fd_r,fd_w) = os.pipe()
scanner = KukurukuScanner(l, confdir) scanner = KukurukuScanner(l)
sdr = top_block(device, scanner.conf.rate, ppm, fd_w) sdr = top_block(device, scanner.conf.rate, ppm, fd_w)
......
...@@ -24,10 +24,13 @@ else: ...@@ -24,10 +24,13 @@ else:
filenames = os.listdir(".") filenames = os.listdir(".")
for filename in filenames: for filename in filenames:
if filename[-6:] != ".cfile":
fn = os.path.basename(filename)
if fn[-6:] != ".cfile":
continue continue
p = filename.split("-") p = fn.split("-")
if len(p) != 8: if len(p) != 8:
continue continue
......
...@@ -13,47 +13,58 @@ import time ...@@ -13,47 +13,58 @@ import time
from gnuradio.filter import firdes from gnuradio.filter import firdes
import struct import struct
from libutil import cfg_safe, engnum
MAINSECTION = "General" MAINSECTION = "General"
class ConfReader(): class ConfReader():
def __init__(self, path): def __init__(self):
confdir = os.path.join(os.path.expanduser('~'), ".kukuruku/scanner/")
# Read global config # Read global config
rc = configparser.ConfigParser() rc = configparser.ConfigParser()
rc.read(os.path.join(path, "main.conf")) cfpath = os.path.join(confdir, "main.conf")
self.bufsize = rc.getint(MAINSECTION, 'bufsize') rc.read(cfpath)
self.rate = rc.getint(MAINSECTION, 'rate')
self.interval = rc.getint(MAINSECTION, 'interval') self.bufsize = cfg_safe(rc.getint, MAINSECTION, 'bufsize', 2048000, cfpath)
self.skip = rc.getint(MAINSECTION, 'skip') self.rate = cfg_safe(rc.getint, MAINSECTION, 'rate', 2048000, cfpath)
self.fftw = rc.getint(MAINSECTION, 'fftw') self.interval = cfg_safe(rc.getint, MAINSECTION, 'interval', 5, cfpath)
self.fftskip = rc.getint(MAINSECTION, 'fftskip') self.skip = cfg_safe(rc.getint, MAINSECTION, 'skip', 3, cfpath)
self.overlap = rc.getfloat(MAINSECTION, 'overlap') self.fftw = cfg_safe(rc.getint, MAINSECTION, 'fftw', 2048, cfpath)
self.nonce = rc.get(MAINSECTION, 'nonce') self.fftskip = cfg_safe(rc.getint, MAINSECTION, 'fftskip', 32, cfpath)
self.floor = rc.getfloat(MAINSECTION, 'floor') self.overlap = cfg_safe(rc.getfloat, MAINSECTION, 'overlap', 0.2, cfpath)
self.sql = rc.getfloat(MAINSECTION, 'sql') self.nonce = cfg_safe(rc.get, MAINSECTION, 'nonce', "abcdef", cfpath)
self.filtermargin = rc.getfloat(MAINSECTION, 'filtermargin') self.floor = cfg_safe(rc.getfloat, MAINSECTION, 'floor', 0.4, cfpath)
self.transition = rc.getfloat(MAINSECTION, 'transition') self.sql = cfg_safe(rc.getfloat, MAINSECTION, 'sql', 0.5, cfpath)
self.minw = rc.getint(MAINSECTION, 'minw')*1000 self.filtermargin = cfg_safe(rc.getfloat, MAINSECTION, 'filtermargin', 1.5, cfpath)
self.maxw = rc.getint(MAINSECTION, 'maxw')*1000 self.transition = cfg_safe(rc.getfloat, MAINSECTION, 'transition', 0.2, cfpath)
self.messgain = rc.getint(MAINSECTION, 'messgain') self.minw = cfg_safe(rc.get, MAINSECTION, 'minw', 10000, cfpath)
self.mingain = rc.getint(MAINSECTION, 'mingain') self.maxw = cfg_safe(rc.get, MAINSECTION, 'maxw', 200000, cfpath)
self.maxgain = rc.getint(MAINSECTION, 'maxgain') self.messgain = cfg_safe(rc.getint, MAINSECTION, 'messgain', 1, cfpath)
self.gain = rc.get(MAINSECTION, 'gain') self.mingain = cfg_safe(rc.getint, MAINSECTION, 'mingain', 10, cfpath)
self.stickactivity = rc.getboolean(MAINSECTION, 'stickactivity') self.maxgain = cfg_safe(rc.getint, MAINSECTION, 'maxgain', 49, cfpath)
self.stick = rc.getint(MAINSECTION, 'stick') self.gain = cfg_safe(rc.get, MAINSECTION, 'gain', "0,30,30,30", cfpath)
self.dumpspectrum = rc.get(MAINSECTION, 'dumpspectrum') self.stickactivity = cfg_safe(rc.getboolean, MAINSECTION, 'stickactivity', False, cfpath)
self.stick = cfg_safe(rc.getint, MAINSECTION, 'stick', 10, cfpath)
self.silencegap = cfg_safe(rc.getint, MAINSECTION, 'silencegap', 5, cfpath)
self.dumpspectrum = cfg_safe(rc.get, MAINSECTION, 'dumpspectrum', "never", cfpath)
self.gainpcs = self.gain.split(",") self.gainpcs = self.gain.split(",")
if len(self.gainpcs) != 4: if len(self.gainpcs) != 4:
print("Wrong gain string format in configuration %s, should be \"N,N,N,N\""%self.gain) print("Wrong gain string format in configuration %s, should be \"N,N,N,N\""%self.gain)
sys.exit(1) sys.exit(1)
if self.messgain < 0 or self.messgain > 3: if self.messgain < -1 or self.messgain > 3:
print("messagin parameter must be an integer from range [0, 3]") print("messagin parameter must be an integer from range [-1, 3]")
sys.exit(1) sys.exit(1)
defaultgain = int(self.gainpcs[self.messgain]) defaultgain = int(self.gainpcs[self.messgain])
files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] self.minw = engnum(self.minw)
self.maxw = engnum(self.maxw)
self.rate = engnum(self.rate)
files = [f for f in os.listdir(confdir) if os.path.isfile(os.path.join(confdir, f))]
self.scanframes = [] self.scanframes = []
self.cronframes = [] self.cronframes = []
...@@ -62,7 +73,7 @@ class ConfReader(): ...@@ -62,7 +73,7 @@ class ConfReader():
# Read scanframe and channel config files # Read scanframe and channel config files
for f in files: for f in files:
if f == "main.conf": if f == "main.conf" or f == "blacklist.conf":
continue continue
if f[-5:] != ".conf": if f[-5:] != ".conf":
continue continue
...@@ -70,32 +81,23 @@ class ConfReader(): ...@@ -70,32 +81,23 @@ class ConfReader():
print("file %s"%f) print("file %s"%f)
rc = configparser.ConfigParser() rc = configparser.ConfigParser()
rc.read(os.path.join(path, f)) rc.read(os.path.join(confdir, f))
try:
floor = rc.getfloat(MAINSECTION, 'floor')
except:
floor = self.floor
try: floor = cfg_safe(rc.getfloat, MAINSECTION, 'floor', self.floor)
sql = rc.getfloat(MAINSECTION, 'sql') sql = cfg_safe(rc.getfloat, MAINSECTION, 'sql', self.sql)
except: stickactivity = cfg_safe(rc.getboolean, MAINSECTION, 'stickactivity', self.stickactivity)
sql = self.sql stick = cfg_safe(rc.getfloat, MAINSECTION, 'stick', self.stick)
silencegap = cfg_safe(rc.getfloat, MAINSECTION, 'silencegap', self.silencegap)
try: if "freqstart" in rc.options(MAINSECTION): # range randscan
stickactivity = rc.getboolean(MAINSECTION, 'stickactivity') freqstart = rc.get(MAINSECTION, "freqstart")
except: freqstart = engnum(freqstart) + step/2
stickactivity = self.stickactivity
try: freqstop = rc.get(MAINSECTION, "freqstop")
stick = rc.getfloat(MAINSECTION, 'stick') freqstop = engnum(freqstop) - step/2
except:
stick = self.stick
if "freqstart" in rc.options(MAINSECTION): # range randscan
freqstart = rc.getint(MAINSECTION, "freqstart")*1000 + step/2
freqstop = rc.getint(MAINSECTION, "freqstop")*1000 - step/2
numf = int(math.ceil(float(freqstop - freqstart)/step)) numf = int(math.ceil(float(freqstop - freqstart)/step))
if numf == 0: # freqstart == freqstop if numf == 0: # freqstart == freqstop
numf = 1 numf = 1
delta = freqstop delta = freqstop
...@@ -108,6 +110,7 @@ class ConfReader(): ...@@ -108,6 +110,7 @@ class ConfReader():
frm.floor = floor