Commit 2e715051 authored by Jenda's avatar Jenda

scanner progress

parent 8063b3a5
../client/ClientStructures.py
\ No newline at end of file
../client/ConfReader.py
\ No newline at end of file
from __future__ import print_function
import os
import sys
import time
import threading
......@@ -9,20 +10,23 @@ import hashlib
import math
import numpy as np
import xlater
import struct
from datetime import datetime
from libutil import Struct, safe_cast
from gnuradio.filter import firdes
def channelhelper():
ChannelHelperT = Struct("channelhelper", "rotator decim rate file taps carry rotpos firpos cylen")
return ChannelHelperT(0.0, 0, 0, [])
return ChannelHelperT()
COMPLEX64 = 8
class scanner():
def __init__(self, l, sdr):
def __init__(self, l, confdir):
self.l = l
self.sdr = sdr
self.conf = util.ConfReader(confdir)
def croncmp(self, i, frag):
if frag == "*":
......@@ -51,7 +55,7 @@ class scanner():
ret = self.croncmp(d.minute, pieces[0]) and self.croncmp(d.hour, pieces[1]) and\
self.croncmp(d.day, pieces[2]) and self.croncmp(d.month, pieces[3]) and\
self.croncmp(d.weekday() + 1, pieces[4])
self.l.l("%s -> %s"%(s, ret))
self.l.l("%s -> %s"%(s, ret), "DBG")
return ret
def find_cronjob(self, cronframes):
......@@ -69,86 +73,183 @@ class scanner():
self.sdr.tune(cronframe.freq)
self.l.l("cron record: tune %i"%cronframe.freq, "INFO")
self.sdrflush()
helpers = []
peaks = []
for channel in cronframe.channels:
peaks.append((channel.freq-cronframe.freq, channel.bw))
self.do_record(peaks, cronframe.cronlen, cronframe.stickactivity, 1, channel.freq, cronframe.floor, None)
def scan(self, scanframe):
''' Find peaks in spectrum, record if specified in allow/blacklist '''
starttime = time.time()
self.sdr.tune(scanframe.freq)
self.sdrflush()
self.l.l("scan: tune %ik"%(scanframe.freq/1000), "INFO")
delta = self.conf.interval - time.time() % self.conf.interval
nbytes = int(self.conf.rate * COMPLEX64 * delta)
nbytes -= nbytes % COMPLEX64
sbuf = self.pipefile.read(nbytes)
acc = self.compute_spectrum(sbuf)
floor = sorted(acc)[int(scanframe.floor * self.conf.fftw)]
peaks = self.find_peaks(acc, floor + scanframe.sql)
# TODO consult blacklist
self.do_record(peaks, scanframe.stick, scanframe.stickactivity, self.conf.filtermargin, scanframe.freq, scanframe.floor, sbuf)
def do_record(self, peaks, stoptime, stickactivity, safetymargin, center, floor, buf):
lastact = time.time()
helpers = []
for peak in peaks:
ch = channelhelper()
ch.decim = math.ceil(self.samplerate / (channel.bw*2))
ch.rate = self.samplerate/ch.decim
ch.file = self.getfn(channel.freq, ch.rate) + ".cfile"
ch.rotator = -float(channel.freq-cronframe.freq)/self.samplerate * 2*math.pi
f = peak[0]
w = peak[1]*safetymargin
ch.decim = math.ceil(self.conf.rate/w)
ch.rate = self.conf.rate/ch.decim
ch.file = os.open(self.getfn(f+center, ch.rate) + ".cfile", os.O_WRONLY|os.O_CREAT)
ch.rotator = -float(f)/self.conf.rate * 2*math.pi
ch.rotpos = np.zeros(2, dtype=np.float32)
ch.rotpos[0] = 1 # start with unit vector
ch.taps = channel.taps
taps = firdes.low_pass(1, self.conf.rate, w/2, w*self.conf.transition, firdes.WIN_HAMMING)
ch.taps = struct.pack("=%if"%len(taps), *taps)
ch.firpos = np.zeros(1, dtype=np.int32)
ch.cylen = len(ch.taps)
ch.carry = '\0' * ch.cylen
self.l.l("Recording %s"%ch.file, "INFO")
helpers.append(ch)
while True:
if time.time() - starttime > cronframe.cronlen:
for ch in helpers:
ch.file.close()
return
# read data from sdr
buf = self.pipefile.read(self.conf.bufsize * COMPLEX64)
if buf is None:
# read data from sdr
buf = self.pipefile.read(self.conf.bufsize * COMPLEX64)
# xlate each channel
for ch in helpers:
# ta vec s carry je nesmysl, udelal bych workery a ty by to zjistovaly podle ch.file
xlater.xdump(buf, len(buf), ch.carry, ch.cylen, ch.taps, len(ch.taps),
int(ch.decim), ch.rotator, ch.rotpos, ch.firpos, ch.file)
ch.carry = buf[-ch.cylen:]
def scan(self, scanframe):
''' Find peaks in spectrum, record if specified in allow/blacklist '''
starttime = time.time()
self.sdr.tune(scanframe.freq)
self.sdrflush()
self.l.l("scan: tune %i"%scanframe.freq, "INFO")
if stickactivity:
acc = self.compute_spectrum(buf)
for peak in peaks:
if self.check_activity(acc, peak, floor):
lastact = time.time()
self.l.l("%f has activity, continuing"%peak[0], "INFO")
delta = self.conf.interval - time.time() % self.conf.interval
sbuf = self.pipefile.read(int(self.samplerate * COMPLEX64 * delta))
iters = 0
if time.time() > lastact + stoptime:
self.l.l("Record stop", "INFO")
break
buf = None
for ch in helpers:
os.close(ch.file)
def check_activity(self, acc, peak, q):
floor = sorted(acc)[int(q * self.conf.fftw)]
binhz = self.conf.rate/self.conf.fftw
startbin = int(peak[0]/binhz - peak[1]/(2*binhz))
stopbin = int(peak[0]/binhz + peak[1]/(2*binhz))
for i in range(startbin, stopbin):
if acc[i] > floor:
return True
return False
def compute_spectrum(self, sbuf):
acc = np.zeros(self.conf.fftw)
iters = 0
dt = np.dtype("=c8")
for i in range(0, len(sbuf)-self.conf.fftw*8, self.conf.fftw*8): # compute short-time FFTs, sum result to acc
for i in range(0, len(sbuf)-self.conf.fftw*COMPLEX64, self.conf.fftw*self.conf.fftskip*COMPLEX64): # compute short-time FFTs, sum result to acc
buf = np.frombuffer(sbuf, count=self.conf.fftw, dtype=dt, offset = i)
buf = buf*self.window
fft = np.absolute(np.fft.fft(buf))
acc += fft
iters += 1
floor = sorted(acc)[int(scanframe.floor * self.conf.fftw)]
acc = np.divide(acc, iters)
acc = np.log(acc)
# FFT yields a list of positive frequencies and then negative frequencies.
# We want it in the "natural" order.
acc = acc.tolist()[len(acc)/2:] + acc.tolist()[:len(acc)/2]
# smooth the result with a simple triangular filter
acc2 = [acc[0]]
for i in range(1, len(acc)-1):
acc2.append((acc[i-1]*0.5 + acc[i] + acc[i+1]*0.5) / 2)
acc2.append(acc[len(acc)-1])
return acc2
print(acc)
def find_peaks(self, acc, floor):
first = -1
print("read %i"%len(buf))
binhz = self.conf.rate/self.conf.fftw
minspan = self.conf.minw/binhz
maxspan = self.conf.maxw/binhz
peaks = []
for i in range(1, len(acc)):
if acc[i] > floor and acc[i-1] < floor:
first = i
if acc[i] < floor and acc[i-1] > floor:
if (i-first) >= minspan and (i-first) <= maxspan:
f = binhz*(((i+first)/2)-self.conf.fftw/2)
w = binhz*(i-first)
peaks.append((f, w))
self.l.l("signal at %f width %f"%(f, w), "INFO")
return peaks
def sdrflush(self):
self.pipefile.read(self.conf.bufsize * COMPLEX64)
def work(self, confdir, outpipe):
self.samplerate = self.sdr.get_sample_rate()
self.pipefile = open(outpipe, "rb")
self.conf = util.ConfReader(confdir, self.samplerate)
def work(self, sdr, file_r):
self.sdr = sdr
self.pipefile = file_r
self.window = np.hamming(self.conf.fftw)
selframe = None
reclen = None
while(True):
delta = self.conf.interval - time.time() % self.conf.interval
if delta < self.conf.skip:
time.sleep(delta)
selframe = self.find_cronjob(self.conf.cronframes)
if selframe:
self.record_long(selframe)
continue
# no cron job now -- pick some frame at random
ctime = "%i"%(math.floor(time.time()) / self.conf.interval)
idx = int(hashlib.sha256(self.conf.nonce + ctime).hexdigest(), 16) % len(self.conf.scanframes)
scanframe = self.conf.scanframes[idx]
self.scan(scanframe)
if len(self.conf.scanframes) > 0:
# no cron job now -- pick some frame at random
ctime = "%i"%(math.floor(time.time()) / self.conf.interval)
idx = int(hashlib.sha256(self.conf.nonce + ctime).hexdigest(), 16) % len(self.conf.scanframes)
scanframe = self.conf.scanframes[idx]
self.scan(scanframe)
else:
time.sleep(delta)
......@@ -3,22 +3,20 @@ SWIG = swig3.0
CFLAGS = -Wall -lm -lvolk -std=gnu99 -O3 -g -ggdb3
all: xlater
all: xlater.py
.PHONY: clean xlater
.PHONY: clean
xlater: v-swig v-wrap v-so
v-swig:
xlater_wrap.c: xlater.c
$(SWIG) -python xlater.i
v-wrap: v-swig
xlater.o: xlater_wrap.c
$(CC) $(CFLAGS) -c -fpic xlater.c xlater_wrap.c -I/usr/include/python2.7
v-so: v-wrap
xlater.py: xlater.o
$(CC) $(CFLAGS) -shared xlater.o xlater_wrap.o -o _xlater.so
clean:
rm later.o _xlater.so xlater_wrap.c xlater_wrap.o
rm -f xlater.o _xlater.so xlater_wrap.c xlater_wrap.o xlater.py xlater.pyc
[General]
freqstart=430000
freqstop=450000
[General]
freq=100000
floor=0.2
sql=1.0
cron=1 5 * * *
sql=0.3
cron=58 21 * * *
cronlen=60
randscan=no
randscan=yes
stickactivity=True
[Channel1]
freq=99700
bw=32
bw=64
continue=30
pipe=/home/jenda/tmp/kukuruku/client/modes/wfm.py -r 0.44
[Channel2]
freq=99300
bw=32
bw=64
continue=30
pipe=/home/jenda/tmp/kukuruku/client/modes/wfm.py -r 0.44
[General]
; size of SDR buffer, used for flushing
bufsize=2048000
; SDR sample rate
rate=2048000
; the duration of one tune, in seconds
interval=10
; do not bother with scan if we have less than this number of seconds left
skip=5
; spectrum transform size
fftw=2048
; transform decimation
; we do transforms on every fftw*fftskip offset
fftskip=32
; when generating scanplan of frequency range, create at least this overlap between tunes
; e.g. if we are scanning from 100 MHz with 1MHz wire SDR, the first scan will be 99.8-100.8,
; the second 100.6 - 101.6 etc.
overlap=0.2
nonce="ahoj"
; when the scanner picks the next frequency, it computes SHA(time+nonce)
; set this to different values if you want multiple scanners jumping independently
nonce="abcdef"
; the noise floor is set as floor-quantile from power spectrum
floor=0.3
sql=1.0
transition=0.1
; treat everything higher than floor+sql as a signal
sql=0.3
; do not care about signals narrower than this (kHz) (e.g. random interference, carrier-only etc.)
minw=10
maxw=100
spacing=12
; do not care about signals wider than this (kHz) (e.g. do not catch 8MHz wide DVB-T multiplex)
maxw=150
; initial gain
gain=0,30,30,30
; try to do autogain with this gain ID parameter
; -1 .. autogain disabled
; 0 .. libosmosdr autogain
; 1 .. libosmosdr RF gain
; 2 .. libosmosdr IF gain
; 3 .. libosmosdr BB gain
; all of them are hardware-specific, if you don't know, you probably want RF gain
messgain=1
; limit the auto gain algorithm -- it should not set gain below and above this
mingain=10
maxgain=49
; when generating FIR filters, make transition band this wide
transition=0.2
; multiply the filnal filter width by this factor
; we usually don't get the exact position of the signal, so we better make the
; filter wider and hope we dump it all...
filtermargin = 1.5
stickactivity = False
stick = 10
from libutil import Struct
def scanframe():
ScanframeT = Struct("scanframe", "freq floor squelch channels gain")
return ScanframeT(None, None, None, [])
ScanframeT = Struct("scanframe", "freq floor squelch channels gain stickactivity pipe")
return ScanframeT(None, None, None, [], 0, False, None)
def cronframe():
CronframeT = Struct("cronframe", "freq floor squelch cronstr cronlen channels gain")
return CronframeT(None, None, None, "", 0, [])
CronframeT = Struct("cronframe", "freq floor squelch cronstr cronlen channels gain stickactivity pipe")
return CronframeT(None, None, None, "", 0, [], 0, False, None)
def channel():
ChannelT = Struct("channel", "freq bw cont")
......
......@@ -18,10 +18,9 @@ import tempfile
class top_block(gr.top_block):
def __init__(self, device, rate, freq, gain, outpipe):
def __init__(self, device, rate, freq, gain, fd):
self.rate = rate
self.gain = gain
self.outpipe = outpipe
gr.top_block.__init__(self, "Top Block")
......@@ -38,18 +37,14 @@ class top_block(gr.top_block):
self.osmosdr_source.set_antenna("", 0)
self.osmosdr_source.set_bandwidth(0, 0)
self.blocks_file_sink = blocks.file_sink(gr.sizeof_gr_complex, outpipe, False)
self.blocks_file_sink.set_unbuffered(False)
self.blocks_file_descriptor_sink_0 = blocks.file_descriptor_sink(gr.sizeof_float*2, fd)
self.connect((self.osmosdr_source, 0), (self.blocks_file_sink, 0))
self.connect((self.osmosdr_source, 0), (self.blocks_file_descriptor_sink_0, 0))
# osmosdr.source.get_sample_rate() seems to be broken
def get_sample_rate(self):
return self.rate
def get_outpipe(self):
return self.outpipe
def tune(self, f):
self.osmosdr_source.set_center_freq(int(f), 0)
......@@ -86,31 +81,25 @@ for opt, arg in opts:
if not confdir:
usage()
#d = tempfile.mkdtemp()
#outpipe = os.path.join(d, 'fifo')
outpipe = "/tmp/pipe"
print("PIPE %s"%outpipe)
#os.mkfifo(outpipe)
(fd_r,fd_w) = os.pipe()
sdr = top_block(device, rate, 0, 0, outpipe)
sdr = top_block(device, rate, 0, 10, fd_w)
l = util.logger()
l.setloglevel("DBG")
scanner = KukurukuScanner.scanner(l, sdr)
scanner = KukurukuScanner.scanner(l, confdir)
file_r = os.fdopen(fd_r, "rb")
t = threading.Thread(target=scanner.work, args = (confdir,outpipe))
t = threading.Thread(target=scanner.work, args = (sdr,file_r))
t.daemon = True
t.start()
sdr.start()
time.sleep(100000)
#try:
# raw_input('Press Enter to quit: ')
#except EOFError:
# pass
#tb.stop()
#tb.wait()
try:
raw_input('Press Enter to quit: ')
except EOFError:
pass
sdr.stop()
......@@ -15,38 +15,45 @@ import struct
MAINSECTION = "General"
class ConfReader():
def __init__(self, path, samplerate):
def __init__(self, path):
# Read global config
rc = configparser.ConfigParser()
rc.read(os.path.join(path, "main.conf"))
self.bufsize = rc.getint(MAINSECTION, 'bufsize')
self.rate = rc.getint(MAINSECTION, 'rate')
self.interval = rc.getint(MAINSECTION, 'interval')
self.skip = rc.getint(MAINSECTION, 'skip')
self.fftw = rc.getint(MAINSECTION, 'fftw')
self.fftskip = rc.getint(MAINSECTION, 'fftskip')
self.overlap = rc.getfloat(MAINSECTION, 'overlap')
self.nonce = rc.get(MAINSECTION, 'nonce')
self.floor = rc.getfloat(MAINSECTION, 'floor')
self.sql = rc.getfloat(MAINSECTION, 'sql')
self.filtermargin = rc.getfloat(MAINSECTION, 'filtermargin')
self.transition = rc.getfloat(MAINSECTION, 'transition')
self.minw = rc.getint(MAINSECTION, 'minw')*1000
self.maxw = rc.getint(MAINSECTION, 'maxw')*1000
self.spacing = rc.getint(MAINSECTION, 'spacing')*1000
self.messgain = rc.getint(MAINSECTION, 'messgain')
self.mingain = rc.getint(MAINSECTION, 'mingain')
self.maxgain = rc.getint(MAINSECTION, 'maxgain')
self.gain = rc.get(MAINSECTION, 'gain')
self.samplerate = samplerate
self.stickactivity = rc.getboolean(MAINSECTION, 'stickactivity')
self.stick = rc.getint(MAINSECTION, 'stick')
files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
self.scanframes = []
self.cronframes = []
step = float(samplerate) * (1-self.overlap)
step = float(self.rate) * (1-self.overlap)
# Read scanframe and channel config files
for f in files:
if f == "main.conf":
continue
if f[-5:] != ".conf":
continue
print("file %s"%f)
......@@ -63,6 +70,21 @@ class ConfReader():
except:
sql = self.sql
try:
stickactivity = rc.getboolean(MAINSECTION, 'stickactivity')
except:
stickactivity = self.stickactivity
try:
stick = rc.getfloat(MAINSECTION, 'stick')
except:
stick = self.stick
try:
pipe = rc.get(MAINSECTION, 'pipe')
except:
pipe = None
if "freqstart" in rc.options(MAINSECTION): # range randscan
freqstart = rc.getint(MAINSECTION, "freqstart")*1000 + step/2
freqstop = rc.getint(MAINSECTION, "freqstop")*1000 - step/2
......@@ -73,6 +95,9 @@ class ConfReader():
frm = scanframe()
frm.freq = cf
frm.floor = floor
frm.stickactivity = stickactivity
frm.stick = stick
frm.pipe = pipe
frm.sql = sql
frm.gain = self.gain
......@@ -85,6 +110,9 @@ class ConfReader():
frm = cronframe()
frm.freq = rc.getint(MAINSECTION, "freq")*1000
frm.floor = "floor"
frm.stickactivity = stickactivity
frm.stick = stick
frm.pipe = pipe
frm.sql = sql
frm.gain = self.gain
frm.cron = rc.get(MAINSECTION, "cron")
......@@ -103,6 +131,9 @@ class ConfReader():
frm = scanframe()
frm.freq = rc.getint(MAINSECTION, "freq")*1000
frm.floor = floor
frm.stickactivity = stickactivity
frm.stick = stick
frm.pipe = pipe
frm.sql = sql
frm.gain = self.gain
......@@ -120,7 +151,7 @@ class ConfReader():
ch = channel()
ch.freq = rc.getint(ssect, "freq")*1000
ch.bw = rc.getint(ssect, "bw")*1000
taps = firdes.low_pass(1, self.samplerate, ch.bw, ch.bw*self.transition, firdes.WIN_HAMMING)
taps