I have the following code sample. The Normal distribution plots perfectly; whereas the Binomial does not.
from sympy import *
from sympy.stats import *
X = Binomial('X', 100 , S.Half, 1, 0) # AttributeError: 'bool' object has no attribute 'evalf'
X = Normal('n1', 50, 5) # plots nicely
x = symbols("x")
d = density(X)(x)
plot(d, (x, 0, 100))
I wonder what I am doing wrong in trying to plot the Binomial.
Ok, this works.
import numpy as np
from sympy import *
from sympy.stats import *
from matplotlib import pyplot
%matplotlib inline
X = Binomial('X', 100 , S.Half, 1, 0)
d = density(X)
u = np.arange(0,100)
v1 = [d(i) for i in u]
pyplot.plot(u,v1)
Related
I have scaled my original data X1:
scaler = StandardScaler()
X1_scaled = pd.DataFrame(scaler.fit_transform(X1),columns = X1.columns)
and then performed k-means clustering:
kmeans = KMeans(
init="random",
n_clusters=3,
n_init=10,
max_iter=300,
random_state=123)
X1['label'] = kmeans.fit_predict(X1_scaled[['Wn', 'LL']])
# get centroids
centroids = kmeans.cluster_centers_
cen_x = [i[0] for i in centroids]
cen_y = [i[1] for i in centroids]
Now, I would like to plot the original data(X1) and the centroids, but the centroids are scaled, so when I plot the results:
g = sns.scatterplot(x=X1.Wn, y= X1.LL, hue=X1.label,
data=X1, palette='colorblind',
legend='full')
g = sns.scatterplot(cen_x,cen_y,s=80,color='black')
the centroids is outside the clusters.
How can I plot the original data, with the groups and the centroids?
this is the image I got:
and this is what I would like to have, but with the original data and not the scaled data:
You can call scaler.inverse_transform() on the centroids. (Note that sns.scatterplot is an axes-level function and returns an ax, not a FacetGrid.)
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
from matplotlib import pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
X1 = pd.DataFrame({'Wn': np.random.rand(30) * 12, 'LL': np.random.rand(30) * 6})
scaler = StandardScaler()
X1_scaled = pd.DataFrame(scaler.fit_transform(X1), columns=X1.columns)
kmeans = KMeans(init="random",
n_clusters=3,
n_init=10,
max_iter=300,
random_state=123)
X1['label'] = kmeans.fit_predict(X1_scaled[['Wn', 'LL']])
# get centroids
centroids = scaler.inverse_transform(kmeans.cluster_centers_)
cen_x = [i[0] for i in centroids]
cen_y = [i[1] for i in centroids]
ax = sns.scatterplot(x='Wn', y='LL', hue='label',
data=X1, palette='colorblind',
legend='full')
sns.scatterplot(x=cen_x, y=cen_y, s=80, color='black', ax=ax)
plt.tight_layout()
plt.show()
This is a scatterplot of my data:
In my opinion, I see two clusters here. A large, wide, highly dense one up top, and a smaller, lower density one in the bottom right:
When I do k-means clustering, the data ends up getting split right down the middle:
My question: Is there a clustering algorithm out there that would be able to discern the clusters that I am seeing/describing?
I don't see any code or data here. Anyway, I'd recommend DBSCAN, Affinity Propagation, and you can try GMM as well. You can find some sample code below.
https://scikit-learn.org/stable/modules/clustering.html#dbscan
https://scikit-learn.org/stable/modules/clustering.html#affinity-propagation
https://scikit-learn.org/stable/modules/mixture.html#mixture
print(__doc__)
import numpy as np
from sklearn.cluster import DBSCAN
from sklearn import metrics
from sklearn.datasets import make_blobs
from sklearn.preprocessing import StandardScaler
# #############################################################################
# Generate sample data
centers = [[1, 1], [-1, -1], [1, -1]]
X, labels_true = make_blobs(n_samples=750, centers=centers, cluster_std=0.4,
random_state=0)
X = StandardScaler().fit_transform(X)
# #############################################################################
# Compute DBSCAN
db = DBSCAN(eps=0.3, min_samples=10).fit(X)
core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
core_samples_mask[db.core_sample_indices_] = True
labels = db.labels_
# Number of clusters in labels, ignoring noise if present.
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
n_noise_ = list(labels).count(-1)
print('Estimated number of clusters: %d' % n_clusters_)
print('Estimated number of noise points: %d' % n_noise_)
print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels_true, labels))
print("Completeness: %0.3f" % metrics.completeness_score(labels_true, labels))
print("V-measure: %0.3f" % metrics.v_measure_score(labels_true, labels))
print("Adjusted Rand Index: %0.3f"
% metrics.adjusted_rand_score(labels_true, labels))
print("Adjusted Mutual Information: %0.3f"
% metrics.adjusted_mutual_info_score(labels_true, labels))
print("Silhouette Coefficient: %0.3f"
% metrics.silhouette_score(X, labels))
# #############################################################################
# Plot result
import matplotlib.pyplot as plt
# Black removed and is used for noise instead.
unique_labels = set(labels)
colors = [plt.cm.Spectral(each)
for each in np.linspace(0, 1, len(unique_labels))]
for k, col in zip(unique_labels, colors):
if k == -1:
# Black used for noise.
col = [0, 0, 0, 1]
class_member_mask = (labels == k)
xy = X[class_member_mask & core_samples_mask]
plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
markeredgecolor='k', markersize=14)
xy = X[class_member_mask & ~core_samples_mask]
plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
markeredgecolor='k', markersize=6)
plt.title('Estimated number of clusters: %d' % n_clusters_)
plt.show()
I want to make a correlation heatmap for a DataFrame and a regression plot for each pair of the variables. I have tried to read all the docs and am still having a very hard time to connect two plots so that when I tap the heatmap, the corresponding regression plot can show up.
Here's some example code:
import holoviews as hv
from holoviews import opts
import seaborn as sns
import numpy as np
import pandas as pd
hv.extension('bokeh')
df = sns.load_dataset('tips')
df = df[['total_bill', 'tip', 'size']]
corr = df.corr()
heatmap = hv.HeatMap((corr.columns, corr.index, corr))\
.opts(tools=['tap', 'hover'], height=400, width=400, toolbar='above')
m, b = np.polyfit(df.tip, df.total_bill, deg=1)
x = np.linspace(df.tip.min(), df.tip.max())
y = m*x + b
curve = hv.Curve((x, y))\
.opts(height=400, width=400, color='red', ylim=(0, 100))
points = hv.Scatter((df.tip, df.total_bill))
hv.Layout((points * curve) + heatmap).cols(2)
I adjusted the relevant parts of the docs http://holoviews.org/reference/streams/bokeh/Tap.html with your code. Maybe this clears up your confusion.
import pandas as pd
import numpy as np
import holoviews as hv
from holoviews import opts
hv.extension('bokeh', width=90)
import seaborn as sns
# Declare dataset
df = sns.load_dataset('tips')
df = df[['total_bill', 'tip', 'size']]
# Declare HeatMap
corr = df.corr()
heatmap = hv.HeatMap((corr.columns, corr.index, corr))
# Declare Tap stream with heatmap as source and initial values
posxy = hv.streams.Tap(source=heatmap, x='total_bill', y='tip')
# Define function to compute histogram based on tap location
def tap_histogram(x, y):
m, b = np.polyfit(df[x], df[y], deg=1)
x_data = np.linspace(df.tip.min(), df.tip.max())
y_data = m*x_data + b
return hv.Curve((x_data, y_data), x, y) * hv.Scatter((df[x], df[y]), x, y)
tap_dmap = hv.DynamicMap(tap_histogram, streams=[posxy])
(heatmap + tap_dmap).opts(
opts.Scatter(height=400, width=400, color='red', ylim=(0, 100), framewise=True),
opts.HeatMap(tools=['tap', 'hover'], height=400, width=400, toolbar='above'),
opts.Curve(framewise=True)
)
Two common problems we face while modeling is collinearity and nonlinearity. The collinearity could be visualized with a correlation heatmap, but it would become hard to explore with a large amount of variables/features. In the following application, you can hover the mouse over to check the correlation coefficient between any two variables. When you tap, the scatter plot will be updated with a second-degree fitted curve to reveal the nonlinearity between the two variables.
With the help of #doopler, I changed the code a little bit and share it here:
import numpy as np
import pandas as pd
import holoviews as hv
hv.extension('bokeh')
# generate random data
df = pd.DataFrame(data={'col_1': np.random.normal(5, 2, 100)})
df['col_2'] = df.col_1 + np.random.gamma(5, 2, 100)
df['col_3'] = df.col_1*2 + np.random.normal(0, 10, 100)
df['col_4'] = df.col_1**2 + np.random.normal(0, 10, 100)
df['col_5'] = np.sin(df.col_1)
df['col_6'] = np.cos(df.col_1)
corr = df.corr().abs()
# mask the upper triangle of the heatmap
corr.values[np.triu_indices_from(corr, 0)] = np.nan
heatmap = hv.HeatMap((corr.columns, corr.index, corr))\
.opts(tools=['hover'], height=400, width=400, fontsize=9,
toolbar='above', colorbar=False, cmap='Blues',
invert_yaxis=True, xrotation=90, xlabel='', ylabel='',
title='Correlation Coefficient Heatmap (absolute value)')
# define tap stream with heatmap as source
tap_xy = hv.streams.Tap(source=heatmap, x='col_1', y='col_4')
# calculate correlation plot based on tap
def tap_corrplot(x, y):
# drop missing values if there are any
df_notnull = df[[x, y]].dropna(how='any')
# fit a 2nd degree line/curve
m1, m2, b = np.polyfit(df_notnull[x], df_notnull[y], deg=2)
# generate data to plot fitted line/curve
x_curve = np.linspace(df[x].min(), df[x].max())
y_curve = m1*x_curve**2 + m2*x_curve+ b
curve = hv.Curve((x_curve, y_curve), x, y)\
.opts(color='#fc4f30', framewise=True)
scatter = hv.Scatter((df[x], df[y]), x, y)\
.opts(height=400, width=400, fontsize=9, size=5,
alpha=0.2, ylim=(df[y].min(), df[y].max()),
color='#30a2da', framewise=True,
title='Correlation Plot (2nd degree fit)')
return curve * scatter
# map tap in heatmap with correlation plot
tap_dmap = hv.DynamicMap(tap_corrplot, streams=[tap_xy])
layout = heatmap + tap_dmap
layout
In case that you need to run a Bokeh application:
from bokeh.server.server import Server
renderer = hv.renderer('bokeh')
app = renderer.app(layout)
server = Server({'/': app}, port=0)
server.start()
server.show('/')
The code works well with Jupyter Lab. If you use Jupyter Notebook, check this link.
So I'm trying to plot this function using PyPlot in julia
here's what I did:
using PyPlot
theta0 = 160
theta = range(0,stop=180,length = 50)
m1 = 4
m2 = range(1,stop = 200,length = 50)
d = pi/180
function km(m,th)
((sqrt((m/m1)^2-sin(th*d)^2)+cos(th*d))/(1+(m)))^2
end
PyPlot.plot(m2,km(m2,theta0))
and I get this error
I can't find anything wrong, please help.
Hello Stackoverflow members,
I have built up an LSTM Seuqential Model for Forex M15 Values, specifically for the pair EURUSD, with typical_price as the price type.
Now after setting up and train the model, I would like to predict, extrapolate the typical_price for one future day.
In my dataset I took the data for one month (January 2017) from 1st to 30th as training and testing dataset (1920 values). Now I would like to extrapolate the prices for the 31th of January. I cannot really resolve what the model likes as input data and shape, to extrapolate the data from the last value of the 30th of January.
Can someone give me a hint or explain what the function model.predict() needs as input values?
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from subprocess import check_output
from keras.layers.core import Dense, Activation, Dropout
from keras.layers.recurrent import LSTM
from keras.models import Sequential
from sklearn.cross_validation import train_test_split
import time #helper libraries
from sklearn.preprocessing import MinMaxScaler
import matplotlib.pyplot as plt
from numpy import newaxis
from keras.metrics import mean_squared_error
from sklearn.model_selection import StratifiedKFold
import time
df = pd.read_csv('EURUSD15.csv')
df.columns = ['date','time','open','high','low','close','vol']
df['date']=df['date'].str.replace('.','-')
J = df[(df['date'] > '2017-01-01') & (df['date'] < '2017-01-30')]
J['timestamp'] = pd.to_datetime(J['date'].apply(str)+' '+J['time'])
J['tp']=((J['high']+J['low']+J['close'])/3)
EURUSD = J[['timestamp','open','high','low','close','vol','tp']]
df = EURUSD.drop(['timestamp','open','high','low','close','vol'], axis=1)
scaler = MinMaxScaler(feature_range=(0,1))
df = scaler.fit_transform(df)
def window_transform_series(series,window_size):
# containers for input/output pairs
dataX = []
datay = []
for i in range(window_size, len(series)):
dataX.append(series[i - window_size:i])
datay.append(series[i])
# reshape
dataX = np.asarray(dataX)
dataX.shape = (np.shape(dataX)[0:2])
datay = np.asarray(datay)
datay.shape = (len(datay),1)
return dataX,datay
window_size = 50
dataX,datay = window_transform_series(series = df, window_size = window_size)
train_test_split = int(np.ceil(2*len(datay)/float(3))) # set the split point
# partition the training set
# X_train = dataX[:train_test_split,:]
# y_train = datay[:train_test_split]
# partition the training set
X_train = dataX[:train_test_split,:]
y_train = datay[:train_test_split]
#keep the last chunk for testing
X_test = dataX[train_test_split:,:]
y_test = datay[train_test_split:]
# NOTE: to use keras's RNN LSTM module our input must be reshaped
X_train = np.asarray(np.reshape(X_train, (X_train.shape[0], window_size, 1)))
X_test = np.asarray(np.reshape(X_test, (X_test.shape[0], window_size, 1)))
import keras
np.random.seed(0)
#Build an RNN to perform regression on our time series input/output data
model = Sequential()
model.add(LSTM(5, input_shape=(window_size, 1)))
model.add(Dense(1))
optimizer = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
# compile the model
model.compile(loss='mean_squared_error', optimizer=optimizer)
model.fit(X_train, y_train, epochs=500, batch_size=64, verbose=1)
train_predict = model.predict(X_train)
test_predict = model.predict(X_test)
# print out training and testing errors
training_error = model.evaluate(X_train, y_train, verbose=0)
print('training error = ' + str(training_error))
testing_error = model.evaluate(X_test, y_test, verbose=0)
print('testing error = ' + str(testing_error))
training error = 0.0001732897365647525
testing error = 0.00019586048660112955
%matplotlib inline
#plot original series
plt.plot(df, color = 'k')
# plot training set prediction
split_pt = train_test_split + window_size
plt.plot(np.arange(window_size,split_pt,1),train_predict,color = 'b')
# plot testing set prediction
plt.plot(np.arange(split_pt,split_pt + len(test_predict),1), test_predict,color ='r')
# pretty up graph
plt.xlabel('day')
plt.ylabel('(normalized) price of EURUSD')
plt.legend(['original series','training fit','testing fit'],loc='center left', bbox_to_anchor=(1, 0.5))
plt.show()
It suppose to be open, highest, lowest price, volume. So you can predict closing price for some imaginary date or you can model.predict(X_test[30]). But one line in your code is strange - the line where you drop all yours features. I wonder how yout X_train[0] looks like.