I'm trying to get some technical ind. with some of those commands in this link:https://github.com/enigmampc/catalyst/blob/master/catalyst/pipeline/factors/equity/technical.py,
but in the quant.notebook I'm not able to get "from numexpr import evaluate", so evaluate is not defined.
How can I solve this?
from numexpr import evaluate
class FastochasticOscillator(CustomFactor):
inputs=(USEquityPricing.close,USEquityPricing.high,USEquityPricing.low)
window_safe=True
window_length=14
def compute(self, today, assets, out, closes, highs, lows):
highest_high= nanmax(highs, axis=0)
lowest_low= nanmin(lows, axis=0)
latest_close= closes[-1]
evaluate(
'((tc - ll) / (hh - ll)) * 100',
local_dict={
'tc':latest_close,
'll':lowest_low,
'hh':highest_high,
},
global_dict={},
out=out,
)
K= FastochasticOscillator(window_length=14)
return Pipeline(columns={
'K':K,
},screen=base)
I'm working on the Quantopian notebook and when I attempt to import it gives me this: InputRejected: Importing evaluate from numexpr raised an ImportError. Did you mean to import errstate from numpy?
Actually I do not find a way to import numexpr on Quantopian but on Jupyter it do not give problems. Therefore the problem is related to the online IDE. Moreover, I simply re-write the FastOsc ind. in another way to use it inside the pipeline in the quantopian online IDE.
class Fast(CustomFactor):
inputs=(USEquityPricing.close,USEquityPricing.high,USEquityPricing.low)
window_length=14
def compute(self, today, assets, out, close, high, low):
highest_high= nanmax(high, axis=0)
lowest_low= nanmin(low, axis=0)
latest_close= close[-1]
out[:]= ((latest_close - lowest_low) / (highest_high - lowest_low)*100)
Related
I follow this example
create the example timetable py file, and put it in the $Home/airflow/plugins
create the example dag file, and put it in $Home/airflow/dags
After restart scheduler and webserver, I get DAG import error. In the web UI, the last line of detailed error message:
airflow.exceptions.SerializationError: Failed to serialize DAG 'example_timetable_dag2': Timetable class 'AfterWorkdayTimetable' is not registered
But if I run airflow plugins, I can see the timetable is in the name and source list.
How to fix this error?
Detail of plugins/AfterWorkdayTimetable.py:
from datetime import timedelta
from typing import Optional
from pendulum import Date, DateTime, Time, timezone
from airflow.plugins_manager import AirflowPlugin
from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction, Timetable
UTC = timezone("UTC")
class AfterWorkdayTimetable(Timetable):
def infer_data_interval(self, run_after: DateTime) -> DataInterval:
weekday = run_after.weekday()
if weekday in (0, 6): # Monday and Sunday -- interval is last Friday.
days_since_friday = (run_after.weekday() - 4) % 7
delta = timedelta(days=days_since_friday)
else: # Otherwise the interval is yesterday.
delta = timedelta(days=1)
start = DateTime.combine((run_after - delta).date(), Time.min).replace(tzinfo=UTC)
return DataInterval(start=start, end=(start + timedelta(days=1)))
def next_dagrun_info(
self,
*,
last_automated_data_interval: Optional[DataInterval],
restriction: TimeRestriction,
) -> Optional[DagRunInfo]:
if last_automated_data_interval is not None: # There was a previous run on the regular schedule.
last_start = last_automated_data_interval.start
last_start_weekday = last_start.weekday()
if 0 <= last_start_weekday < 4: # Last run on Monday through Thursday -- next is tomorrow.
delta = timedelta(days=1)
else: # Last run on Friday -- skip to next Monday.
delta = timedelta(days=(7 - last_start_weekday))
next_start = DateTime.combine((last_start + delta).date(), Time.min).replace(tzinfo=UTC)
else: # This is the first ever run on the regular schedule.
next_start = restriction.earliest
if next_start is None: # No start_date. Don't schedule.
return None
if not restriction.catchup:
# If the DAG has catchup=False, today is the earliest to consider.
next_start = max(next_start, DateTime.combine(Date.today(), Time.min).replace(tzinfo=UTC))
elif next_start.time() != Time.min:
# If earliest does not fall on midnight, skip to the next day.
next_day = next_start.date() + timedelta(days=1)
next_start = DateTime.combine(next_day, Time.min).replace(tzinfo=UTC)
next_start_weekday = next_start.weekday()
if next_start_weekday in (5, 6): # If next start is in the weekend, go to next Monday.
delta = timedelta(days=(7 - next_start_weekday))
next_start = next_start + delta
if restriction.latest is not None and next_start > restriction.latest:
return None # Over the DAG's scheduled end; don't schedule.
return DagRunInfo.interval(start=next_start, end=(next_start + timedelta(days=1)))
class WorkdayTimetablePlugin(AirflowPlugin):
name = "workday_timetable_plugin"
timetables = [AfterWorkdayTimetable]
Details of dags/test_afterwork_timetable.py:
import datetime
from airflow import DAG
from AfterWorkdayTimetable import AfterWorkdayTimetable
from airflow.operators.dummy import DummyOperator
with DAG(
dag_id="example_workday_timetable",
start_date=datetime.datetime(2021, 1, 1),
timetable=AfterWorkdayTimetable(),
tags=["example", "timetable"],
) as dag:
DummyOperator(task_id="run_this")
If I run airflow plugins:
name | source
==================================+==========================================
workday_timetable_plugin | $PLUGINS_FOLDER/AfterWorkdayTimetable.py
I had similar issue.
Either you need to add __init__.py file or you should try this to debug your issue:
Get all plugin manager objects:
from airflow import plugins_manager
plugins_manager.initialize_timetables_plugins()
plugins_manager.timetable_classes
I got this result: {'quarterly.QuarterlyTimetable': <class 'quarterly.QuarterlyTimetable'>}
Compare your result with exception message. If timetable_classes dictionary has a different plugin name you should either change plugin file path.
You could also try this inside DAG python file:
from AfterWorkdayTimetable import AfterWorkdayTimetable
from airflow import plugins_manager
print(plugins_manager.as_importable_string(AfterWorkdayTimetable))
This would help you find the name that airflow tries to use when searching through timetable_classes dictionary.
You need to register the timetable in "timetables" array via plugin interface. See:
https://airflow.apache.org/docs/apache-airflow/stable/plugins.html
Encountered the same issue.
These are the steps I followed.
Add the Timetable file(custom_tt.py) into plugins folder.
Make sure the plugin folder has _ _ init_ _.py file present in plugins folder.
Change the lazy_load_plugins in airflow.cfg to False.
lazy_load_plugins = False
Add import statement in dagfile as:
from custom_tt import CustomTimeTable
In DAG as
DAG(timetable=CustomTimeTable())
Restart the webserver and scheduler.
Problem fixed.
They have found the resolution to this but doesn't seem like they have updated the documentation to represent the fix just yet.
Your function
def infer_data_interval(self, run_after: DateTime) -> DataInterval:
should be
def infer_manual_data_interval(self, *, run_after: DateTime) -> DataInterval:
See reference:
Apache airflow.timetables.base Documentation
After updating the function with the correct name and extra parameter, everything else should work for you as it did for me.
I was running into this as well. airflow plugins reports that the plugin is registered, and running the DAG script on the command line works fine, but the web UI reports that the plugin is not registered. #Bakuchi's answer pointed me in the right direction.
In my case, the problem was how I was importing the Timetable - airflow apparently expects you to import it relative to the $PLUGINS_FOLDER, not from any other directory, even if that other directory is also on the PYTHONPATH.
For a concrete example:
export PYTHONPATH=/path/to/my/code:$PYTHONPATH
# airflow.cfg
plugins_folder = /path/to/my/code/airflow_plugins
# dag.py
import sys
from airflow_plugins.custom_timetable import CustomTimetable as Bad
from custom_timetable import CustomTimetable as Good
from airflow import plugins_manager
plugins_manager.initialize_timetables_plugins()
print(sys.path) # /path/to/my/code:...:/path/to/my/code/airflow_plugins
print(plugins_manager.as_importable_string(Bad)) # airflow_plugins.custom_timetable.CustomTimetable
print(plugins_manager.as_importable_string(Good)) # custom_timetable.CustomTimetable
print(plugins_manager.timetable_classes) # {'custom_timetable.CustomTimetable': <class 'custom_timetable.CustomTimetable'>}
A bad lookup in plugins_manager.timetable_classes is ultimately what ends up raising the _TimetableNotRegistered error, so the fix is to make the keys match by changing how the timetable is imported.
I submitted a bug report: https://github.com/apache/airflow/issues/21259
I'm using Hydra for training machine learning models. It's great for doing complex commands like python train.py data=MNIST batch_size=64 loss=l2. However, if I want to then run the trained model with the same parameters, I have to do something like python reconstruct.py --config_file path_to_previous_job/.hydra/config.yaml. I then use argparse to load in the previous yaml and use the compose API to initialize the Hydra environment. The path to the trained model is inferred from the path to Hydra's .yaml file. If I want to modify one of the parameters, I have to add additional argparse parameters and run something like python reconstruct.py --config_file path_to_previous_job/.hydra/config.yaml --batch_size 128. The code then manually overrides any Hydra parameters with those that were specified on the command line.
What's the right way of doing this?
My current code looks something like the following:
train.py:
import hydra
#hydra.main(config_name="config", config_path="conf")
def main(cfg):
# [training code using cfg.data, cfg.batch_size, cfg.loss etc.]
# [code outputs model checkpoint to job folder generated by Hydra]
main()
reconstruct.py:
import argparse
import os
from hydra.experimental import initialize, compose
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('hydra_config')
parser.add_argument('--batch_size', type=int)
# [other flags and parameters I may need to override]
args = parser.parse_args()
# Create the Hydra environment.
initialize()
cfg = compose(config_name=args.hydra_config)
# Since checkpoints are stored next to the .hydra, we manually generate the path.
checkpoint_dir = os.path.dirname(os.path.dirname(args.hydra_config))
# Manually override any parameters which can be changed on the command line.
batch_size = args.batch_size if args.batch_size else cfg.data.batch_size
# [code which uses checkpoint_dir to load the model]
# [code which uses both batch_size and params in cfg to set up the data etc.]
This is my first time posting, so let me know if I should clarify anything.
If you want to load the previous config as is and not change it, use OmegaConf.load(file_path).
If you want to re-compose the config (and it sounds like you do, because you added that you want override things), I recommend that you use the Compose API and pass in parameters from the overrides file in the job output directory (next to the stored config.yaml), but concatenate the current run parameters.
This script seems to be doing the job:
import os
from dataclasses import dataclass
from os.path import join
from typing import Optional
from omegaconf import OmegaConf
import hydra
from hydra import compose
from hydra.core.config_store import ConfigStore
from hydra.core.hydra_config import HydraConfig
from hydra.utils import to_absolute_path
# You can also use a yaml config file instead of this Structured Config
#dataclass
class Config:
load_checkpoint: Optional[str] = None
batch_size: int = 16
loss: str = "l2"
cs = ConfigStore.instance()
cs.store(name="config", node=Config)
#hydra.main(config_path=".", config_name="config")
def my_app(cfg: Config) -> None:
if cfg.load_checkpoint is not None:
output_dir = to_absolute_path(cfg.load_checkpoint)
original_overrides = OmegaConf.load(join(output_dir, ".hydra/overrides.yaml"))
current_overrides = HydraConfig.get().overrides.task
hydra_config = OmegaConf.load(join(output_dir, ".hydra/hydra.yaml"))
# getting the config name from the previous job.
config_name = hydra_config.hydra.job.config_name
# concatenating the original overrides with the current overrides
overrides = original_overrides + current_overrides
# compose a new config from scratch
cfg = compose(config_name, overrides=overrides)
# train
print("Running in ", os.getcwd())
print(OmegaConf.to_yaml(cfg))
if __name__ == "__main__":
my_app()
~/tmp$ python train.py
Running in /home/omry/tmp/outputs/2021-04-19/21-23-13
load_checkpoint: null
batch_size: 16
loss: l2
~/tmp$ python train.py load_checkpoint=/home/omry/tmp/outputs/2021-04-19/21-23-13
Running in /home/omry/tmp/outputs/2021-04-19/21-23-22
load_checkpoint: /home/omry/tmp/outputs/2021-04-19/21-23-13
batch_size: 16
loss: l2
~/tmp$ python train.py load_checkpoint=/home/omry/tmp/outputs/2021-04-19/21-23-13 batch_size=32
Running in /home/omry/tmp/outputs/2021-04-19/21-23-28
load_checkpoint: /home/omry/tmp/outputs/2021-04-19/21-23-13
batch_size: 32
loss: l2
Hello this is my first question (iam noob in everything) and iam kind of nervous not to get on your nervouse.
My overall goal is to create a 30 min lesson (part) where pupils can see:
interactive graphics and the super-fency CODE.
get attracted to become software engineers_innen or at least start to read the code. (me Measure the impact)
For this i think Jupyter notebook is the best weapon. For usability reason and different levels of interactivity i started to think about creating html-files to have remote access (my admins will kill me before letting me install sth. more on the school enviroment)
So i saw this:
from bokeh.plotting import figure
from bokeh.resources import CDN
from bokeh.embed import file_html
plot = figure()
plot.circle([1,2], [3,4])
html = file_html(plot, CDN, "my plot")
to create a html out of a bokeh figure.
But how to get nice interact sliders to this standalone html-file.
So how to (even if its non-sense) convert this example:
from ipywidgets import interact
import numpy as np
from bokeh.io import push_notebook, show, output_notebook
from bokeh.plotting import figure
output_notebook()
In [ ]:
x = np.linspace(0, 2*np.pi, 2000)
y = np.sin(x)
In [ ]:
p = figure(title="simple line example", plot_height=300, plot_width=600, y_range=(-5,5))
r = p.line(x, y, color="#2222aa", line_width=3)
In [ ]:
def update(f, w=1, A=1, phi=0):
if f == "sin": func = np.sin
elif f == "cos": func = np.cos
elif f == "tan": func = np.tan
r.data_source.data['y'] = A * func(w * x + phi)
push_notebook()
In [ ]:
show(p, notebook_handle=True)
In [ ]:
interact(update, f=["sin", "cos", "tan"], w=(0,100), A=(1,5), phi=(0, 20, 0.1))
Into sth. like this: https://demo.bokeh.org/sliders
Thanks for every comment. sry my english is not as good as look like, but iam not as smart as i look like
There's not any straightforward way that I know of to make this work in standalone HTML with Jupyter interactors. However, you can make standalone output that accomplishes this with Bokeh's own widgets:
https://docs.bokeh.org/en/latest/docs/gallery/slider.html
There is some ongoing work that should be out with Bokeh 2.0 that will allow Jupyter interactors to work with Bokeh server apps (not standalone output).
How can I use Asynchronous Widgets on jupyter lab?
I'm trying to reproduce the official Asynchronous Widgets-Example on jupyter lab, but the await never continues.
Setup / reproduction
docker run --rm -p 8888:8888 -e JUPYTER_ENABLE_LAB=yes jupyter/datascience-notebook start-notebook.sh --NotebookApp.token=''
firefox 0.0.0.0:8888
create a new python3 notebook
create a cell and enter the code below
run cell
move slider
code for the cell
%gui asyncio
import asyncio
def wait_for_change(widget, value):
future = asyncio.Future()
def getvalue(change):
# make the new value available
future.set_result(change.new)
widget.unobserve(getvalue, value)
widget.observe(getvalue, value)
return future
from ipywidgets import IntSlider
slider = IntSlider()
async def f():
for i in range(10):
print('did work %s'%i)
#x = await asyncio.sleep(1)
x = await wait_for_change(slider, 'value')
print('async function continued with value %s'%x)
asyncio.ensure_future(f())
#task = asyncio.create_task(f())
slider
Expected result
The cell outputs
did work 0
async function continued with value 1
did work 1
async function continued with value 2
[...]
Actual output
nothing after the first did work 0
Notes
I'm specifically talking about jupyter lab and not about regular jupyter notebooks
There is no error-message or anything. The expected output just doesn't happen
The minimal asyncio-example does work in jupyter lab:
import asyncio
async def main():
print('hello')
await asyncio.sleep(1)
print('world')
await main()
when you leave out the -e JUPYTER_ENABLE_LAB=yes, then you get a regular jupyter notebook without jupyter lab and the expected result happens.
this is not a duplicate of ipywidgets widgets values not changing or Jupyter Interactive Widget not executing properly, because these questions nether include jupyter lab nor asyncio
Actually it works, but jupyter lose print output.
Try this code:
from IPython.display import display
import ipywidgets as widgets
out = widgets.Output()
import asyncio
def wait_for_change(widget, value):
future = asyncio.Future()
def getvalue(change):
# make the new value available
future.set_result(change.new)
widget.unobserve(getvalue, value)
widget.observe(getvalue, value)
return future
from ipywidgets import IntSlider
slider = IntSlider()
# Now the key: the container is displayed (while empty) in the main thread
async def f():
for i in range(10):
out.append_stdout('did work %s'%i)
x = await wait_for_change(slider, 'value')
out.append_stdout('async function continued with value %s'%x)
asyncio.ensure_future(f())
display(slider)
display(out)
You can find more details here: https://github.com/jupyter-widgets/ipywidgets/issues/2567#issuecomment-535971252
I've had luck with jupyter-ui-poll to synchronize widget activity with the Jupyter Python kernal:
https://github.com/Kirill888/jupyter-ui-poll
In particular I used it here:
https://github.com/AaronWatters/jp_doodle/blob/master/jp_doodle/auto_capture.py
Works for me.
Hope that helps!
I need to find all python file in folder excluding __init__.py
My first attempt was
import re
search_path.rglob(re.compile("(?!__init__).*.py"))
Such code fails, so i end up with:
filter(
lambda path: '__init__.py' != path.name and path.name.endswith('.py') and path.is_file(), search_path.rglob("*.py")
)
Looks like rglob does not support python regexps.
Why?
Does rglob supports negative patterns?
Can this code be more elegant?
I need something very much like this too. I came up with this:
import pathlib
search_path = pathlib.Path.cwd() / "test_folder"
for file in filter(lambda item: item.name != "__init__.py", search_path.rglob("./*.py")):
print(f"{file}")
Alternatively, you can use fnmatch.
import pathlib
import fnmatch
search_path = pathlib.Path.cwd() / "test_folder"
for file in search_path.rglob("./*.py"):
if not fnmatch.fnmatch(file.name, "__init__.py"):
print(f"{file}")