properly implemented social planner's problem

julia_code/BellmanResidualMinimization.jl

@@ -16,7 +16,7 @@ md"""
 
 # ╔═╡ 9fa41b7c-1923-4c1e-bfc6-20ce4a1a2ede
 md"""
-Number of Constellations: $(const N_constellations = 3)
+Number of Constellations: $(const N_constellations = 4)
 
 Number of Debris Trackers: $(const N_debris = 1)
 
@@ -74,7 +74,7 @@ function survival(
 		,physical_model::BasicModel
 	)	
     return exp.(
-		-physical_model.satellite_collision_rates * stocks 
+		-(physical_model.satellite_collision_rates .- physical_model.decay_rate) * stocks 
 		.- (physical_model.debris_collision_rate*debris)
 	)
 end
@@ -127,12 +127,12 @@ function value_function_generator(number_params=10)
 		        )
 		        ,Flux.Chain(
 					Flux.Dense(N_debris, N_states,Flux.relu)
-		            ,Flux.Dense(N_states, N_states)
+		            ,Flux.Dense(N_states, N_states,Flux.σ)
 				)
 			)
 		    #Apply some transformations to the preprocessed data.
 			,Flux.Dense(N_states*3,number_params,Flux.σ)
-			,Flux.Dense(number_params,1,Flux.σ)
+			,Flux.Dense(number_params,1)
 		)
 
 end
@@ -153,7 +153,7 @@ function policy_function_generator(number_params=10)
 			)
 		    #Apply some transformations
 			,Flux.Dense(N_states*3,number_params,Flux.σ)
-			,Flux.Dense(number_params,N_constellations,Flux.σ)
+			,Flux.Dense(number_params,N_constellations,Flux.relu)
 		)
 
 end
@@ -187,18 +187,18 @@ begin
 
 end
 
+# ╔═╡ 340da189-f443-4376-a82d-7699a21ab7a2
+abstract type EconomicParameters end
+
 # ╔═╡ 206ac4cc-5102-4381-ad8a-777b02dc4d5a
-begin
+begin #basic linear model
-	abstract type EconomicParameters end
 	struct EconModel1 <: EconomicParameters
 		β::Real
 		payoff_array::Array{Real}
 		policy_costs::Array{Real}
 	end
-end
 
-# ╔═╡ 1cbaa2e5-55e4-46f9-82d0-04b481470094
+	function payoff1(
-function payoff1(
 	s::Vector
 	,d::Vector
 	,a::Vector
@@ -207,6 +207,27 @@ function payoff1(
 	return em.payoff_array*s - em.policy_costs*a
 end
 
+end
+
+# ╔═╡ eebb8706-a431-4fd1-b7a5-40f07a63d5cb
+begin #basic CES model
+		struct CESParams <: EconomicParameters
+			β::Real
+			r::Real #elasticity of subsititution
+			payoff_array::Array{Real}
+			policy_costs::Array{Real}
+			debris_costs::Array{Real}
+		end
+	function CES_with_debris(
+		s::Vector
+		,d::Vector
+		,a::Vector
+		,em::CESParams
+	)
+		return (em.payoff_array*(s.^em.r) - em.debris_costs*(d.^em.r)).^(1/em.r) - em.policy_costs*a
+	end
+end
+
 # ╔═╡ f8d582cb-10cf-4c72-8127-787f662e0567
 #=
 This struct organizes information about a given constellation operator
@@ -225,7 +246,44 @@ md"""
 
 # ╔═╡ b433a7ec-8264-48d6-8b95-53d2ec4bad05
 md"""
-# Testing
+# Setup examples of parameter models
+"""
+
+# ╔═╡ 65e0b1fa-d5e1-4ff6-8736-c9d6b5f40150
+em1 = EconModel1(0.95, [1 0 0 0], [5 0 0 0])
+#=
+This is the most basic profit model
+
+
+You earn 1 per operating satellite and it costs 5 per launch.
+Only interaction is through debris.
+=#
+
+# ╔═╡ 19ccfc3a-6dbb-4c64-bf03-e2e219ef0efe
+begin
+	em2_a = EconModel1(0.95, [1 -0.02 -0.02 0], [5 0 0 0])
+	em2_b = EconModel1(0.95, [-0.02 1 -0.02 0], [0 5 0 0])
+	em2_c = EconModel1(0.95, [0 -0.02 1 -0.02], [0 0 5 0])
+	em2_d = EconModel1(0.95, [0 -0.02 -0.02 1], [0 0 0 5])
+	#=
+	This is a simple addition to the basic model, where you lose some benefit based 
+	the size of your competitor's satellites.
+	Constellations interact throuch debris and imposing costs on one another.
+	=#
+end
+
+# ╔═╡ dc614254-c211-4552-b985-03020bfc5ab3
+em3 = CESParams(0.95,0.6,[1 0 0 0], [5 0 0 0], Vector([0.002]))
+#=
+This is a variation on a CES model.
+
+The model is CES the relationship between payoffs and debris.
+In this particular specification, the only interaction is in debris
+=#
+
+# ╔═╡ cd55e232-493d-4849-8bd7-b0ba85e21bab
+md"""
+# Start setting things up
 """
 
 # ╔═╡ fb6aacff-c42d-4ec1-88cb-5ce1b2e8874f
@@ -249,15 +307,13 @@ function Ξ(
 	return sum([bellman_residuals.^2 maximization_condition])
 end
 
-# ╔═╡ 65e0b1fa-d5e1-4ff6-8736-c9d6b5f40150
-em1 = EconModel1(0.95, [1 0 0 ], [5 0 0 ])
-
 # ╔═╡ f30904a7-5caa-449a-a5bd-f2aa78777a9a
 begin
 	#setup the operators
-	operators = [ ConstellationOperator(payoff1,em1,value_function_generator())
+	operators = [ ConstellationOperator(payoff1,em2_a,value_function_generator())
-			,ConstellationOperator(payoff1,em1,value_function_generator())
+			,ConstellationOperator(payoff1,em2_b,value_function_generator())
-			,ConstellationOperator(payoff1,em1,value_function_generator())
+			,ConstellationOperator(payoff1,em2_c,value_function_generator())
+			,ConstellationOperator(payoff1,em2_d,value_function_generator())
 	]
 
 	#check whether or not we've matched the setup correctly.
@@ -265,7 +321,7 @@ begin
 end
 
 # ╔═╡ 43b99708-0052-4b78-886c-92ac2b532f29
-begin
+begin #testing
 	s1 = ones(N_constellations)
 	d1 = ones(N_debris)
 	Ξ(s1,d1,bm,operators[1])
@@ -289,13 +345,13 @@ md"""
 
 # ╔═╡ a20959be-65e4-4b69-9521-503bc59f0854
 begin
-	N=20 #increase later
+	N=200 #increase later
 	data = [(rand(1:500, N_constellations),rand(1:500, N_debris)) for n=1:N]
 end
 
 # ╔═╡ 6bf8d29a-7990-4e91-86e6-d9894ed3db27
 #optimizer
-ADAM = Flux.Optimise.ADAM(0.01)
+ADAM = Flux.Optimise.ADAM(0.1)
 
 # ╔═╡ e7ee1a0f-ab9b-439e-a7be-4a6d3b8f160d
 begin
@@ -308,13 +364,15 @@ end
 
 # ╔═╡ 74f5fde3-0593-46fc-a688-f1db7ab28c64
 # Social planners problem
-for epoch in 1:20
+for epoch in 1:200
+	data1 = [(rand(1:500, N_constellations),rand(1:500, N_debris)) for n=1:N]
+	
 	#train the social planner's policy funciton
-	Flux.Optimise.train!(planners_loss, params(policy), data, ADAM)
+	Flux.Optimise.train!(planners_loss, params(policy), data1, ADAM)
 	
 	#Sweep through training the value functions
 	for co in operators
-		Flux.Optimise.train!(planners_loss, params(co.value), data, ADAM)
+		Flux.Optimise.train!(planners_loss, params(co.value), data1, ADAM)
 	end
 end
 
@@ -323,10 +381,32 @@ begin
 	accum = 0.0
 	for d in data
 		accum += planners_loss(d...)
+		
 	end
 	accum/N
 end
 
+# ╔═╡ c50b1d39-fe87-441b-935c-c5fe971d09ef
+policy(data[3])
+
+# ╔═╡ 14e61097-f28f-4029-b6b4-5fb119620fc3
+begin
+	n=1
+	
+	[operators[1].value(data[n])
+	,operators[2].value(data[n])
+	,operators[3].value(data[n])
+	,operators[4].value(data[n])]
+end
+
+# ╔═╡ bf0c6061-daf4-45ac-82bc-b26e093ac6a7
+with_terminal() do
+	for d in data
+		println(d)
+		println("\t",policy(d))
+	end
+end
+
 # ╔═╡ 00000000-0000-0000-0000-000000000001
 PLUTO_PROJECT_TOML_CONTENTS = """
 [deps]
@@ -901,23 +981,27 @@ uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 # ╟─66f0e667-d722-4e1e-807b-84a39cbc41b1
 # ╠═9fa41b7c-1923-4c1e-bfc6-20ce4a1a2ede
 # ╟─90446134-4e45-471c-857d-4e165e51937a
-# ╟─5b45b29e-f0f4-41e9-91e7-d444687feb4e
+# ╠═5b45b29e-f0f4-41e9-91e7-d444687feb4e
 # ╟─152f3a3c-a565-41bb-8e59-6ab0d2315ffb
 # ╟─25ac9438-2b1d-4f6b-9ff1-1695e1d52b51
 # ╠═29ff1777-d276-4e8f-8582-4ca191f2e2ff
-# ╠═f7aabe43-9a2c-4fe0-8099-c29cdf66566c
+# ╟─f7aabe43-9a2c-4fe0-8099-c29cdf66566c
-# ╠═d816b252-bdca-44ba-ac5c-cb21163a1e9a
+# ╟─d816b252-bdca-44ba-ac5c-cb21163a1e9a
 # ╠═95bfc9d8-8427-41d6-9f0f-f155296eef91
+# ╠═340da189-f443-4376-a82d-7699a21ab7a2
 # ╠═206ac4cc-5102-4381-ad8a-777b02dc4d5a
-# ╠═1cbaa2e5-55e4-46f9-82d0-04b481470094
+# ╠═eebb8706-a431-4fd1-b7a5-40f07a63d5cb
 # ╠═f8d582cb-10cf-4c72-8127-787f662e0567
 # ╠═5946daa3-4608-43f3-8933-dd3eb3f4541c
 # ╠═41271ab4-1ec7-431f-9efb-0f7c3da2d8b4
 # ╠═43b99708-0052-4b78-886c-92ac2b532f29
 # ╠═dff642d9-ec5a-4fed-a059-6c07760a3a58
-# ╟─b433a7ec-8264-48d6-8b95-53d2ec4bad05
+# ╠═b433a7ec-8264-48d6-8b95-53d2ec4bad05
-# ╠═fb6aacff-c42d-4ec1-88cb-5ce1b2e8874f
 # ╠═65e0b1fa-d5e1-4ff6-8736-c9d6b5f40150
+# ╠═19ccfc3a-6dbb-4c64-bf03-e2e219ef0efe
+# ╠═dc614254-c211-4552-b985-03020bfc5ab3
+# ╠═cd55e232-493d-4849-8bd7-b0ba85e21bab
+# ╠═fb6aacff-c42d-4ec1-88cb-5ce1b2e8874f
 # ╠═f30904a7-5caa-449a-a5bd-f2aa78777a9a
 # ╟─5abebc1a-370c-4f5f-8826-dc0b143d5166
 # ╠═a20959be-65e4-4b69-9521-503bc59f0854
@@ -925,5 +1009,8 @@ uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 # ╠═e7ee1a0f-ab9b-439e-a7be-4a6d3b8f160d
 # ╠═74f5fde3-0593-46fc-a688-f1db7ab28c64
 # ╠═02f3fe78-e7a7-453f-9ddf-acddf08d8676
+# ╠═c50b1d39-fe87-441b-935c-c5fe971d09ef
+# ╠═14e61097-f28f-4029-b6b4-5fb119620fc3
+# ╠═bf0c6061-daf4-45ac-82bc-b26e093ac6a7
 # ╟─00000000-0000-0000-0000-000000000001
 # ╟─00000000-0000-0000-0000-000000000002