I am currently working on a personal project with pigpio and piscope on raspberry PI 4.
I try to simulate my TV remote by sending IR signal through an IR LED setup connected on GPIO 23 and GND pin (setup is a simple IR LED with a 200 ohm resistor)
I searched on LIRC database my TV remote config file and I did not find it, but I found another one (MKJ40653802-TV) which is said to be working also for my TV which is a LG 50PS3000:
https://www.remote-control-world.eu/lg-c-2_64/lg-mkj42519615-replacement-remote-control-p-4195
also config file :
begin remote
name MKJ40653802-TV
bits 16
flags SPACE_ENC|CONST_LENGTH
eps 30
aeps 100
header 9061 4473
one 591 1660
zero 591 521
ptrail 590
pre_data_bits 16
pre_data 0x20DF
gap 108029
toggle_bit_mask 0x0
begin codes
KEY_POWER 0x10EF # Was: power
After reading LIRC documentation and explainations on how to contruct an IR signal, I managed to get my hands through a python script which create IR waveform to be fired through IR LED
https://github.com/bschwind/ir-slinger/blob/master/pyslinger.py
I simply changed the NEC protocol paramters to the values present in the config file.
Also my power on/off hex value is 0x20DF23DC (pre-data + command) that I convert to binary 32 bits :
00100000110111110010001111011100
my code below :
#!/usr/bin/env python3
# Python IR transmitter
# Requires pigpio library
# Supports NEC, RC-5 and raw IR.
# Danijel Tudek, Aug 2016
import subprocess
import ctypes
import time
# This is the struct required by pigpio library.
# We store the individual pulses and their duration here. (In an array of these structs.)
class Pulses_struct(ctypes.Structure):
_fields_ = [("gpioOn", ctypes.c_uint32),
("gpioOff", ctypes.c_uint32),
("usDelay", ctypes.c_uint32)]
# Since both NEC and RC-5 protocols use the same method for generating waveform,
# it can be put in a separate class and called from both protocol's classes.
class Wave_generator():
def __init__(self,protocol):
self.protocol = protocol
MAX_PULSES = 12000 # from pigpio.h
Pulses_array = Pulses_struct * MAX_PULSES
self.pulses = Pulses_array()
self.pulse_count = 0
def add_pulse(self, gpioOn, gpioOff, usDelay):
self.pulses[self.pulse_count].gpioOn = gpioOn
self.pulses[self.pulse_count].gpioOff = gpioOff
self.pulses[self.pulse_count].usDelay = usDelay
self.pulse_count += 1
# Pull the specified output pin low
def zero(self, duration):
self.add_pulse(0, 1 << self.protocol.master.gpio_pin, duration)
# Protocol-agnostic square wave generator
def one(self, duration):
period_time = 1000000.0 / self.protocol.frequency
on_duration = int(round(period_time * self.protocol.duty_cycle))
off_duration = int(round(period_time * (1.0 - self.protocol.duty_cycle)))
total_periods = int(round(duration/period_time))
total_pulses = total_periods * 2
# Generate square wave on the specified output pin
for i in range(total_pulses):
if i % 2 == 0:
self.add_pulse(1 << self.protocol.master.gpio_pin, 0, on_duration)
else:
self.add_pulse(0, 1 << self.protocol.master.gpio_pin, off_duration)
# NEC protocol class
class NEC():
def __init__(self,
master,
frequency=38000,
duty_cycle=0.5,
leading_pulse_duration=9061,
leading_gap_duration=4473,
one_pulse_duration = 591,
one_gap_duration = 1660,
zero_pulse_duration = 591,
zero_gap_duration = 521,
trailing_pulse = [1, 590]):
self.master = master
self.wave_generator = Wave_generator(self)
self.frequency = frequency # in Hz, 38000 per specification
self.duty_cycle = duty_cycle # duty cycle of high state pulse
# Durations of high pulse and low "gap".
# The NEC protocol defines pulse and gap lengths, but we can never expect
# that any given TV will follow the protocol specification.
self.leading_pulse_duration = leading_pulse_duration # in microseconds, 9000 per specification
self.leading_gap_duration = leading_gap_duration # in microseconds, 4500 per specification
self.one_pulse_duration = one_pulse_duration # in microseconds, 562 per specification
self.one_gap_duration = one_gap_duration # in microseconds, 1686 per specification
self.zero_pulse_duration = zero_pulse_duration # in microseconds, 562 per specification
self.zero_gap_duration = zero_gap_duration # in microseconds, 562 per specification
self.trailing_pulse = trailing_pulse # trailing 562 microseconds pulse, some remotes send it, some don't
print("NEC protocol initialized")
# Send AGC burst before transmission
def send_agc(self):
print("Sending AGC burst")
self.wave_generator.one(self.leading_pulse_duration)
self.wave_generator.zero(self.leading_gap_duration)
# Trailing pulse is just a burst with the duration of standard pulse.
def send_trailing_pulse(self):
print("Sending trailing pulse")
self.wave_generator.one(self.trailing_pulse[1])
# This function is processing IR code. Leaves room for possible manipulation
# of the code before processing it.
def process_code(self, ircode):
if (self.leading_pulse_duration > 0) or (self.leading_gap_duration > 0):
self.send_agc()
for i in ircode:
if i == "0":
self.zero()
elif i == "1":
self.one()
else:
print("ERROR! Non-binary digit!")
return 1
if self.trailing_pulse[0] == 1:
self.send_trailing_pulse()
return 0
# Generate zero or one in NEC protocol
# Zero is represented by a pulse and a gap of the same length
def zero(self):
self.wave_generator.one(self.zero_pulse_duration)
self.wave_generator.zero(self.zero_gap_duration)
# One is represented by a pulse and a gap three times longer than the pulse
def one(self):
self.wave_generator.one(self.one_pulse_duration)
self.wave_generator.zero(self.one_gap_duration)
# RC-5 protocol class
# Note: start bits are not implemented here due to inconsistency between manufacturers.
# Simply provide them with the rest of the IR code.
class RC5():
def __init__(self,
master,
frequency=36000,
duty_cycle=0.33,
one_duration=889,
zero_duration=889):
self.master = master
self.wave_generator = Wave_generator(self)
self.frequency = frequency # in Hz, 36000 per specification
self.duty_cycle = duty_cycle # duty cycle of high state pulse
# Durations of high pulse and low "gap".
# Technically, they both should be the same in the RC-5 protocol, but we can never expect
# that any given TV will follow the protocol specification.
self.one_duration = one_duration # in microseconds, 889 per specification
self.zero_duration = zero_duration # in microseconds, 889 per specification
print("RC-5 protocol initialized")
# This function is processing IR code. Leaves room for possible manipulation
# of the code before processing it.
def process_code(self, ircode):
for i in ircode:
if i == "0":
self.zero()
elif i == "1":
self.one()
else:
print("ERROR! Non-binary digit!")
return 1
return 0
# Generate zero or one in RC-5 protocol
# Zero is represented by pulse-then-low signal
def zero(self):
self.wave_generator.one(self.zero_duration)
self.wave_generator.zero(self.zero_duration)
# One is represented by low-then-pulse signal
def one(self):
self.wave_generator.zero(self.one_duration)
self.wave_generator.one(self.one_duration)
# RAW IR ones and zeroes. Specify length for one and zero and simply bitbang the GPIO.
# The default values are valid for one tested remote which didn't fit in NEC or RC-5 specifications.
# It can also be used in case you don't want to bother with deciphering raw bytes from IR receiver:
# i.e. instead of trying to figure out the protocol, simply define bit lengths and send them all here.
class RAW():
def __init__(self,
master,
frequency=36000,
duty_cycle=0.33,
one_duration=520,
zero_duration=520):
self.master = master
self.wave_generator = Wave_generator(self)
self.frequency = frequency # in Hz
self.duty_cycle = duty_cycle # duty cycle of high state pulse
self.one_duration = one_duration # in microseconds
self.zero_duration = zero_duration # in microseconds
def process_code(self, ircode):
for i in ircode:
if i == "0":
self.zero()
elif i == "1":
self.one()
else:
print("ERROR! Non-binary digit!")
return 1
return 0
# Generate raw zero or one.
# Zero is represented by low (no signal) for a specified duration.
def zero(self):
self.wave_generator.zero(self.zero_duration)
# One is represented by pulse for a specified duration.
def one(self):
self.wave_generator.one(self.one_duration)
class IR():
def __init__(self, gpio_pin, protocol, protocol_config):
print("Starting IR")
print("Loading libpigpio.so")
self.pigpio = ctypes.CDLL('libpigpio.so')
print("Initializing pigpio")
PI_OUTPUT = 1 # from pigpio.h
self.pigpio.gpioInitialise()
subprocess.Popen('piscope', shell=True)
time.sleep(1)
self.gpio_pin = gpio_pin
print("Configuring pin %d as output" % self.gpio_pin)
self.pigpio.gpioSetMode(self.gpio_pin, PI_OUTPUT) # pin 17 is used in LIRC by default
print("Initializing protocol")
if protocol == "NEC":
self.protocol = NEC(self, **protocol_config)
elif protocol == "RC-5":
self.protocol = RC5(self, **protocol_config)
elif protocol == "RAW":
self.protocol = RAW(self, **protocol_config)
else:
print("Protocol not specified! Exiting...")
return 1
print("IR ready")
# send_code takes care of sending the processed IR code to pigpio.
# IR code itself is processed and converted to pigpio structs by protocol's classes.
def send_code(self, ircode):
print("Processing IR code: %s" % ircode)
code = self.protocol.process_code(ircode)
if code != 0:
print("Error in processing IR code!")
return 1
clear = self.pigpio.gpioWaveClear()
print(clear)
if clear != 0:
print("Error in clearing wave!")
return 1
pulses = self.pigpio.gpioWaveAddGeneric(self.protocol.wave_generator.pulse_count, self.protocol.wave_generator.pulses)
if pulses < 0:
print("Error in adding wave!")
return 1
wave_id = self.pigpio.gpioWaveCreate()
# Unlike the C implementation, in Python the wave_id seems to always be 0.
if wave_id >= 0:
print("Sending wave...")
result = self.pigpio.gpioWaveTxSend(wave_id, 0)
if result >= 0:
print("Success! (result: %d)" % result)
else:
print("Error! (result: %d)" % result)
return 1
else:
print("Error creating wave: %d" % wave_id)
return 1
while self.pigpio.gpioWaveTxBusy():
time.sleep(0.1)
print("Deleting wave")
self.pigpio.gpioWaveDelete(wave_id)
print("Terminating pigpio")
self.pigpio.gpioTerminate()
# Simply define the GPIO pin, protocol (NEC, RC-5 or RAW) and
# override the protocol defaults with the dictionary if required.
# Provide the IR code to the send_code() method.
# An example is given below.
if __name__ == "__main__":
protocol = "NEC"
gpio_pin = 23
protocol_config = dict(one_pulse_duration = 591,
zero_pulse_duration = 591)
ir = IR(gpio_pin, protocol, protocol_config)
ir.send_code("00100000110111110001000011101111")
print("Exiting IR")
When launching the script it's working, I can see the IR LED blinking through phone cam and also I see the waveform generating through piscope :
Everything looks correct to me but I don't know why it's not powering on my TV...
Could you please help me with this problem ? I don't know if I missed something or if I am using the wrong TV code...
Thanks a lot !
I tried other remote code, I tried the toggle-bit-mask on the first bit (toggle_bit_mask = 0x0)
I tried other codes (on and off) from this page :
https://gist.github.com/francis2110/8f69843dd57ae07dce80
with no success
It's working.
I just had to get close to tv (less than 1 meter away).
So I am reviewing my LED setup adding a transistor.
As seen online it should be working from longer distances...
Related
I'm working on a buzzer that has to make a simple sound. But I can't figure out about how to make this buzzer work in MicroPython. I've already tried the following code but it is not doing much. I'm a little bit stuck at this.
import pycom
import machine
import time
from machine import Pin
import board
import pulseio
buzzer = pulseio.PWMOut(board.D16, variable_frequency=True)
def main():
buzzer.duty_cycle = ON
buzzer.frequency = 440
buzzer.duty_cycle = OFF
if __name__ == "__main__":
main()
For more information about what I use as materials:
Buzzer:https://datasheet4u.com/datasheet-pdf/Ningbo/KPT-1410/pdf.php?id=868269
The materials for my project is this:
https://pycom.io/product/lopy4/
https://pycom.io/product/expansion-board-3-0/
I think i'm getting close to it, but can't figure it out.
here is similar function to map function that you have used in arduino
def remap(value, leftMin, leftMax, rightMin, rightMax):
# Figure out how 'wide' each range is
leftSpan = leftMax - leftMin
rightSpan = rightMax - rightMin
# Convert the left range into a 0-1 range (float)
valueScaled = float(value - leftMin) / float(leftSpan)
# Convert the 0-1 range into a value in the right range.
return rightMin + (valueScaled * rightSpan)
for example, if your ADC values are between 2000 and 1000 (on purpose i show that higher ADC value mean lower actual value) you can use function above with following call to get percentage value:
percent_value = remap(
adc_pin.read(),
1000, 2200, # these values comes from experiment above when wet and dry
100, 1
)
and look at the samples in documentation how to make buzz
import array
import pulseio
import pwmio
import board
# 50% duty cycle at 38kHz.
pwm = pwmio.PWMOut(board.D13, frequency=38000, duty_cycle=32768)
pulse = pulseio.PulseOut(pwm)
# on off on off on
pulses = array.array('H', [65000, 1000, 65000, 65000, 1000])
pulse.send(pulses)
# Modify the array of pulses.
pulses[0] = 200
pulse.send(pulses)
I have somewhat a general question for more experienced programmers. I'm somewhat new to programming, but still enjoy it quite a bit. I've been working with Python, and decided to try to program a tic tac toe game, but with variable board size that can be decided by the user (all the way up to a 26x26 board). Here's what I've got so far:
print("""Welcome to tic tac toe!
You will begin by determining who goes first;
Player 2 will decide on the board size, from 3 to 26.
Depending on the size of the board, you will have to
get a certain amount of your symbol (X/O) in a row to win.
To place symbols on the board, input their coordinates;
letter first, then number (e.g. a2, g10, or f18).
That's it for the rules. Good luck!\n""")
while True:
ready = input("Are you ready? When you are, input 'yes'.")
if ready.lower() == 'yes': break
def printboard(n, board):
print() #print board in ranks of n length; n given later
boardbyrnk = [board[ind:ind+n] for ind in range(0,n**2,n)]
for rank in range(n):
rn = f"{n-rank:02d}" #pads with a 0 if rank number < 10
print(f"{rn}|{'|'.join(boardbyrnk[rank])}|") #with rank#'s
print(" ",end="") #files at bottom of board
for file in range(97,n+97): print(" "+chr(file), end="")
print()
def sqindex(prompt, n, board, syms): #takes input & returns index
#ss is a list/array of all possible square names
ss = [chr(r+97)+str(f+1) for r in range(n) for f in range(n)]
while True: #all bugs will cause input to be taken for same turn
sq = input(prompt)
if not(sq in ss): print("Square doesn't exist!"); continue
#the index is found by multiplying rank and adding file #'s
index = n*(n-int(sq[1:])) + ord(sq[0])-97
if board[index] in syms: #ensure it contains ' '
print("The square must be empty!"); continue
return index
def checkwin(n, w, board, sm): #TODO
#check rows, columns and diagonals in terms of n and w;
#presumably return True if each case is met
return False
ps = ["Player 1", "Player 2"]; syms = ['X', 'O']
#determines number of symbols in a row needed to win later on
c = {3:[3,3],4:[4,6],5:[7,13],6:[14,18],7:[19,24],8:[25,26]}
goagain = True
while goagain:
#decide on board size
while True:
try: n=int(input(f"\n{ps[1]}, how long is the board side? "))
except ValueError: print("Has to be an integer!"); continue
if not(2<n<27): print("Has to be from 3 to 26."); continue
break
board = (n**2)*" " #can be rewritten around a square's index
for num in c:
if c[num][0] <= n <= c[num][1]: w = num; break
print(f"You'll have to get {w} symbols in a row to win.")
for tn in range(n**2): #tn%2 = 0 or 1, determining turn order
printboard(n, board)
pt = ps[tn%2]
sm = syms[tn%2]
prompt = f"{pt}, where do you place your {sm}? "
idx = sqindex(prompt, n, board, syms)
#the index found in the function is used to split the board string
board = board[:idx] + sm + board[idx+1:]
if checkwin(n, w, board, sm):
printboard(n, board); print('\n' + pt + ' wins!!\n\n')
break
if board.count(" ") == 0:
printboard(n, board); print("\nIt's a draw!")
while True: #replay y/n; board size can be redetermined
rstorq = input("Will you play again? Input 'yes' or 'no': ")
if rstorq in ['yes', 'no']:
if rstorq == 'no': goagain = False
break
else: print("Please only input lowercase 'yes' or 'no'.")
print("Thanks for playing!")
So my question to those who know what they're doing is how they would recommend determining whether the current player has won (obviously I have to check in terms of w for all cases, but how to program it well?). It's the only part of the program that doesn't work yet. Thanks!
You can get the size of the board from the board variable (assuming a square board).
def winning_line(line, symbol):
return all(cell == symbol for cell in line)
def diag(board):
return (board[i][i] for i in range(len(board)))
def checkwin(board, symbol):
if any(winning_line(row, symbol) for row in board):
return True
transpose = list(zip(*board))
if any(winning_line(column, symbol) for column in transpose):
return True
return any(winning_line(diag(layout), symbol) for layout in (board, transpose))
zip(*board) is a nice way to get the transpose of your board. If you imagine your original board list as a list of rows, the transpose will be a list of columns.
I have downloaded a tensorflow GraphDef that implements a VGG16 ConvNet, which I use doing this :
Pl['images'] = tf.placeholder(tf.float32,
[None, 448, 448, 3],
name="images") #batch x width x height x channels
with open("tensorflow-vgg16/vgg16.tfmodel", mode='rb') as f:
fileContent = f.read()
graph_def = tf.GraphDef()
graph_def.ParseFromString(fileContent)
tf.import_graph_def(graph_def, input_map={"images": Pl['images']})
Besides, I have image features that are homogeneous to the output of the "import/pool5/".
How can I tell my graph that don't want to use his input "images", but the tensor "import/pool5/" as input ?
Thank's !
EDIT
OK I realize I haven't been very clear. Here is the situation:
I am trying to use this implementation of ROI pooling, using a pre-trained VGG16, which I have in the GraphDef format. So here is what I do:
First of all, I load the model:
tf.reset_default_graph()
with open("tensorflow-vgg16/vgg16.tfmodel",
mode='rb') as f:
fileContent = f.read()
graph_def = tf.GraphDef()
graph_def.ParseFromString(fileContent)
graph = tf.get_default_graph()
Then, I create my placeholders
images = tf.placeholder(tf.float32,
[None, 448, 448, 3],
name="images") #batch x width x height x channels
boxes = tf.placeholder(tf.float32,
[None,5], # 5 = [batch_id,x1,y1,x2,y2]
name = "boxes")
And I define the output of the first part of the graph to be conv5_3/Relu
tf.import_graph_def(graph_def,
input_map={'images':images})
out_tensor = graph.get_tensor_by_name("import/conv5_3/Relu:0")
So, out_tensor is of shape [None,14,14,512]
Then, I do the ROI pooling:
[out_pool,argmax] = module.roi_pool(out_tensor,
boxes,
7,7,1.0/1)
With out_pool.shape = N_Boxes_in_batch x 7 x 7 x 512, which is homogeneous to pool5. I would then like to feed out_pool as an input to the op that comes just after pool5, so it would look like
tf.import_graph_def(graph.as_graph_def(),
input_map={'import/pool5':out_pool})
But it doesn't work, I have this error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-89-527398d7344b> in <module>()
5
6 tf.import_graph_def(graph.as_graph_def(),
----> 7 input_map={'import/pool5':out_pool})
8
9 final_out = graph.get_tensor_by_name("import/Relu_1:0")
/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/importer.py in import_graph_def(graph_def, input_map, return_elements, name, op_dict)
333 # NOTE(mrry): If the graph contains a cycle, the full shape information
334 # may not be available for this op's inputs.
--> 335 ops.set_shapes_for_outputs(op)
336
337 # Apply device functions for this op.
/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/ops.py in set_shapes_for_outputs(op)
1610 raise RuntimeError("No shape function registered for standard op: %s"
1611 % op.type)
-> 1612 shapes = shape_func(op)
1613 if len(op.outputs) != len(shapes):
1614 raise RuntimeError(
/home/hbenyounes/vqa/roi_pooling_op_grad.py in _roi_pool_shape(op)
13 channels = dims_data[3]
14 print(op.inputs[1].name, op.inputs[1].get_shape())
---> 15 dims_rois = op.inputs[1].get_shape().as_list()
16 num_rois = dims_rois[0]
17
/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/tensor_shape.py in as_list(self)
745 A list of integers or None for each dimension.
746 """
--> 747 return [dim.value for dim in self._dims]
748
749 def as_proto(self):
TypeError: 'NoneType' object is not iterable
Any clue ?
It is usually very convenient to use tf.train.export_meta_graph to store the whole MetaGraph. Then, upon restoring you can use tf.train.import_meta_graph, because it turns out that it passes all additional arguments to the underlying import_scoped_meta_graph which has the input_map argument and utilizes it when it gets to it's own invocation of import_graph_def.
It is not documented, and took me waaaay toooo much time to find it, but it works!
What I would do is something along those lines:
-First retrieve the names of the tensors representing the weights and biases of the 3 fully connected layers coming after pool5 in VGG16.
To do that I would inspect [n.name for n in graph.as_graph_def().node].
(They probably look something like import/locali/weight:0, import/locali/bias:0, etc.)
-Put them in a python list:
weights_names=["import/local1/weight:0" ,"import/local2/weight:0" ,"import/local3/weight:0"]
biases_names=["import/local1/bias:0" ,"import/local2/bias:0" ,"import/local3/bias:0"]
-Define a function that look something like:
def pool5_tofcX(input_tensor, layer_number=3):
flatten=tf.reshape(input_tensor,(-1,7*7*512))
tmp=flatten
for i in xrange(layer_number):
tmp=tf.matmul(tmp, graph.get_tensor_by_name(weights_name[i]))
tmp=tf.nn.bias_add(tmp, graph.get_tensor_by_name(biases_name[i]))
tmp=tf.nn.relu(tmp)
return tmp
Then define the tensor using the function:
wanted_output=pool5_tofcX(out_pool)
Then you are done !
Jonan Georgiev provided an excellent answer here. The same approach was also described with little fanfare at the end of this git issue: https://github.com/tensorflow/tensorflow/issues/3389
Below is a copy/paste runnable example of using this approach to switch out a placeholder for a tf.data.Dataset get_next tensor.
import tensorflow as tf
my_placeholder = tf.placeholder(dtype=tf.float32, shape=1, name='my_placeholder')
my_op = tf.square(my_placeholder, name='my_op')
# Save the graph to memory
graph_def = tf.get_default_graph().as_graph_def()
print('----- my_op before any remapping -----')
print([n for n in graph_def.node if n.name == 'my_op'])
tf.reset_default_graph()
ds = tf.data.Dataset.from_tensors(1.0)
next_tensor = tf.data.make_one_shot_iterator(ds).get_next(name='my_next_tensor')
# Restore the graph with a custom input mapping
tf.graph_util.import_graph_def(graph_def, input_map={'my_placeholder': next_tensor}, name='')
print('----- my_op after remapping -----')
print([n for n in tf.get_default_graph().as_graph_def().node if n.name == 'my_op'])
Output, where we can clearly see that the input to the square operation has changed.
----- my_op before any remapping -----
[name: "my_op"
op: "Square"
input: "my_placeholder"
attr {
key: "T"
value {
type: DT_FLOAT
}
}
]
----- my_op after remapping -----
[name: "my_op"
op: "Square"
input: "my_next_tensor"
attr {
key: "T"
value {
type: DT_FLOAT
}
}
]
I'm a beginner with TF
I've tried to adapt a code which is working well with some other data (noMNIST) to some new data, and i have a dimensionality error, and i don't know how to deal with it.
To debug, i'm trying to use tf.shape method but it doesn't give me the info i need...
def reformat(dataset, labels):
#dataset = dataset.reshape((-1, num_var)).astype(np.float32)
# Map 2 to [0.0, 1.0, 0.0 ...], 3 to [0.0, 0.0, 1.0 ...]
labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)
return dataset, labels
train_dataset, train_labels = reformat(train_dataset, train_labels)
valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)
test_dataset, test_labels = reformat(test_dataset, test_labels)
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels.shape)
print('Test set', test_dataset.shape, test_labels.shape)
type(train_dataset)
Training set (790184, 29) (790184, 39) Validation set (43899, 29)
(43899, 39) Test set (43899, 29) (43899, 39)
# Adding regularization to the 1 hidden layer network
graph1 = tf.Graph()
batch_size = 128
num_steps=3001
import datetime
startTime = datetime.datetime.now()
def define_and_run_batch(beta):
num_RELU =1024
with graph1.as_default():
# Input data. For the training data, we use a placeholder that will be fed
# at run time with a training minibatch.
tf_train_dataset = tf.placeholder(tf.float32,
shape=(batch_size, num_var))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Variables.
weights_RELU = tf.Variable(
tf.truncated_normal([num_var, num_RELU]))
print(tf.shape(weights_RELU) )
biases_RELU = tf.Variable(tf.zeros([num_RELU]))
weights_layer1 = tf.Variable(
tf.truncated_normal([num_RELU, num_labels]))
biases_layer1 = tf.Variable(tf.zeros([num_labels]))
# Training computation.
logits_RELU = tf.matmul(tf_train_dataset, weights_RELU) + biases_RELU
RELU_vec = tf.nn.relu(logits_RELU)
logits_layer = tf.matmul(RELU_vec, weights_layer1) + biases_layer1
# loss = tf.reduce_mean(
# tf.nn.softmax_cross_entropy_with_logits(logits_layer, tf_train_labels))
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits_layer, tf_train_labels,name="cross_entropy")
l2reg = tf.reduce_sum(tf.square(weights_RELU))+tf.reduce_sum(tf.square(weights_layer1))
# beta = 0.005
loss = tf.reduce_mean(cross_entropy+beta*l2reg)
# Optimizer.
optimizer = tf.train.GradientDescentOptimizer(0.3).minimize(loss)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits_layer)
print("ok")
print(tf.shape(weights_RELU) )
valid_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu((tf.matmul(tf_valid_dataset, weights_RELU) + biases_RELU)),weights_layer1)+biases_layer1)
test_prediction =tf.nn.softmax(
tf.matmul(tf.nn.relu((tf.matmul(tf_test_dataset, weights_RELU) + biases_RELU)),weights_layer1)+biases_layer1)
with tf.Session(graph=graph1) as session:
tf.initialize_all_variables().run()
print("Initialized")
for step in range(num_steps):
# Pick an offset within the training data, which has been randomized.
# Note: we could use better randomization across epochs.
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
# Generate a minibatch.
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
# Prepare a dictionary telling the session where to feed the minibatch.
# The key of the dictionary is the placeholder node of the graph to be fed,
# and the value is the numpy array to feed to it.
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
#
_, l, predictions, logits = session.run(
[optimizer, loss,train_prediction,logits_RELU], feed_dict=feed_dict)
if (step % 500 == 0):
print("Minibatch loss at step %d: %f" % (step, l))
print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
print("Validation accuracy: %.1f%%" % accuracy(
valid_prediction.eval(), valid_labels))
test_acc = accuracy(test_prediction.eval(), test_labels)
print("Test accuracy: %.1f%%" % test_acc)
print('loss=%s' % l)
x = datetime.datetime.now() - startTime
print(x)
return(test_acc,round(l,5))
define_and_run_batch(0.005)
Tensor("Shape:0", shape=(2,), dtype=int32) ok Tensor("Shape_1:0",
shape=(2,), dtype=int32)
--------------------------------------------------------------------------- ValueError Traceback (most recent call
last) in ()
94 return(test_acc,round(l,5))
95
---> 96 define_and_run_batch(0.005)
in define_and_run_batch(beta)
54 print(tf.shape(weights_RELU) )
55 valid_prediction = tf.nn.softmax(
---> 56 tf.matmul(tf.nn.relu((tf.matmul(tf_valid_dataset, weights_RELU) + biases_RELU)),weights_layer1)+biases_layer1)
57
58
/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.pyc
in matmul(a, b, transpose_a, transpose_b, a_is_sparse, b_is_sparse,
name)
949 transpose_a=transpose_a,
950 transpose_b=transpose_b,
--> 951 name=name)
952
953 sparse_matmul = gen_math_ops._sparse_mat_mul
/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.pyc in _mat_mul(a, b, transpose_a, transpose_b, name)
684 """
685 return _op_def_lib.apply_op("MatMul", a=a, b=b, transpose_a=transpose_a,
--> 686 transpose_b=transpose_b, name=name)
687
688
/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/ops/op_def_library.pyc
in apply_op(self, op_type_name, name, **keywords)
653 op = g.create_op(op_type_name, inputs, output_types, name=scope,
654 input_types=input_types, attrs=attr_protos,
--> 655 op_def=op_def)
656 outputs = op.outputs
657 return _Restructure(ops.convert_n_to_tensor(outputs), output_structure)
/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/framework/ops.pyc
in create_op(self, op_type, inputs, dtypes, input_types, name, attrs,
op_def, compute_shapes, compute_device) 2040
original_op=self._default_original_op, op_def=op_def) 2041 if
compute_shapes:
-> 2042 set_shapes_for_outputs(ret) 2043 self._add_op(ret) 2044
self._record_op_seen_by_control_dependencies(ret)
/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/framework/ops.pyc
in set_shapes_for_outputs(op) 1526 raise RuntimeError("No
shape function registered for standard op: %s" 1527
% op.type)
-> 1528 shapes = shape_func(op) 1529 if len(op.outputs) != len(shapes): 1530 raise RuntimeError(
/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/ops/common_shapes.pyc
in matmul_shape(op)
87 inner_a = a_shape[0] if transpose_a else a_shape[1]
88 inner_b = b_shape[1] if transpose_b else b_shape[0]
---> 89 inner_a.assert_is_compatible_with(inner_b)
90 return [tensor_shape.TensorShape([output_rows, output_cols])]
91
/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/framework/tensor_shape.pyc
in assert_is_compatible_with(self, other)
92 if not self.is_compatible_with(other):
93 raise ValueError("Dimensions %s and %s are not compatible"
---> 94 % (self, other))
95
96 def merge_with(self, other):
ValueError: Dimensions Dimension(29) and Dimension(30) are not
compatible
the whole code is on my github
https://github.com/FaguiCurtain/Kaggle-SF
the Udacity Assignment 3 file is working
the original data is here
https://www.kaggle.com/c/sf-crime/data
in Udacity, the data were images and each image was a 28x28 matrix which was reformatted into flattened vectors of size 784
in the Kaggle-SF file, i am feeding vectors of size 29, and labels can take 39 different values.
thanks for your help
In debug mode you can check shapes of you Tensors.
by the way you error is valid_prediction assignment. to make it better for debugging and reading it's better to define each step in a separate line. you are using 4 operation in 1 line. BTW in debug mode (for example in Pycharm) you can inspect the element and check what is causing the problem
To check the dimensions, you can directly print the tensors. When you print the tensors, you can view the dimensions. I suggest if you are a beginner, try using the 'tf.layers' package which contains high level wrappers for the various layers one would need to build a CNN in tensorflow. By using this, you can avoid having to deal with the various low level operations like 'matmul' and adding bias for example. The activations can also be directly applied by the layers without having to implement it manually.
As far as debugging is concerned, from the code since you have merged the operations, its hard to see what is going on under the hood unless we can use a proper debugger. If you are not using an IDE, I suggest using 'pudb'.
I'm writing a DigitalMicrograph script to acquire a mapping data of scattered electron intensities obtained from such as an ADF STEM detector, under various incident-beam conditions being controlled by a handmade script. But, unfortunately I don't know a command to obtain STEM detector signals under non-STEM mode (controlled by DigiScan). What command should I use in this case?
It will be appreciated if you share some wisdom. Thank you very much in advance.
.
As the STEM detector signal is processed by the DigiScan unit, there is no way to read the detector 'signal' independently of it.
Also: You do not get the signal as a "stream" in time, but clocked by the DigiScan. That is, you have to start an acquisition with the DigiScan and can not only "listen" to the detector without one.
However, the DigiScan acquisition is not tied to being in STEM mode. You can start a DigiScan acquisition while being in TEM mode. You can choose the parameters such that acquiring an 'image' is scanning the beam only within a very small area, so that the beam becomes quasi-stationary. Maybe this can help you out?
Here is an example of what I mean: However, I have not tested this on hardware:
// Create "Scan" parameters for an overview
// This image will stay as survey. Its content is not important
// as you're in TEM mode, but we need it as reference
number paramID
number width = 1024 // pixel
number height = 1024 // pixel
number rotation = 0 // degree
number pixelTime= 2 // microseconds
number lSynch = 0 // no-linesync
paramID = DSCreateParameters( width, height, rotation, pixelTime, lSynch )
number signalIndex, dataDepth, selected, imageID
signalIndex = 0 // HAADF (most likely) ?
dataDepth = 4 // 4 byte data
selected = 1 // acquire this signal
imageID = 0 // create new image
DSSetParametersSignal( paramID, signalIndex, dataDepth, selected, imageID )
number continuous = 0 // 0 = single frame, 1 = continuous
number synchronous = 1 // 0 = return immediately, 1 = return when finished
// Capture the "survey" image
DSStartAcquisition( paramID, continuous, synchronous )
image survey := DSGetLastAcquiredImage( signalIndex )
survey.SetName("Survey")
if ( !DSIsValidDSImage( survey ) ) Throw( "Something wrong..")
DSDeleteParameters( paramID ) // remove parameters from memory
// Now we create a "subscan" image for a quasi-stationary beam...
// The size has a minimum size (16x16?) but as we keep the beam
// "stationary" this will rather define your "time-resolution" of
// data. Scan 'speed' is taken from our reference...
number sizeX = 1024
number sizeY = 1024
image Static := IntegerImage( "Static", dataDepth, 0, sizeX, sizeY )
Static.ShowImage()
// defeine "ROI" on the survey. Just the center pixel!
number t,l,b,r
t = height/2
l = width/2
b = t + 1
r = l + 1
DSScanSubRegion( survey, Static, t, l, b, r )