[{"data":1,"prerenderedAt":888},["ShallowReactive",2],{"book-hierarchy":3,"content-query-93eWTrgs1i":199},[4,109],{"key":5,"slug":5,"title":6,"cleanTitle":6,"numberedTitle":6,"description":7,"path":7,"sourcePath":7,"aliases":8,"basePath":9,"filePath":7,"pageType":7,"hidden":10,"effectiveHidden":10,"order":-4,"depth":11,"number":7,"ownAudience":7,"audiences":12,"coverDeLight":7,"coverEnLight":7,"coverDeDark":7,"coverEnDark":7,"children":14},"_info","Info","",[],"/_info",false,1,[13],"Sek I",[15,31,45,59],{"key":16,"slug":17,"title":18,"cleanTitle":18,"numberedTitle":18,"description":19,"path":20,"sourcePath":21,"aliases":22,"basePath":24,"filePath":25,"pageType":26,"hidden":10,"effectiveHidden":10,"order":27,"depth":28,"number":7,"ownAudience":13,"audiences":29,"coverDeLight":7,"coverEnLight":7,"coverDeDark":7,"coverEnDark":7,"children":30},"_info/10_datenschutz","10_datenschutz","Privacy Policy","Information on data protection and the processing of personal data.","/_info/privacy-policy/en","/_info/10_datenschutz/en",["Reactive",23],[21],"/_info/10_datenschutz","content/_Info/10_Datenschutz/en.md","content",10,2,[13],[],{"key":32,"slug":33,"title":34,"cleanTitle":34,"numberedTitle":34,"description":35,"path":36,"sourcePath":37,"aliases":38,"basePath":40,"filePath":41,"pageType":26,"hidden":10,"effectiveHidden":10,"order":42,"depth":28,"number":7,"ownAudience":13,"audiences":43,"coverDeLight":7,"coverEnLight":7,"coverDeDark":7,"coverEnDark":7,"children":44},"_info/11_impressum","11_impressum","Legal Notice","Legal notice for the interactive book.","/_info/legal-notice/en","/_info/11_impressum/en",["Reactive",39],[37],"/_info/11_impressum","content/_Info/11_Impressum/en.md",11,[13],[],{"key":46,"slug":47,"title":48,"cleanTitle":48,"numberedTitle":48,"description":49,"path":50,"sourcePath":51,"aliases":52,"basePath":54,"filePath":55,"pageType":26,"hidden":10,"effectiveHidden":10,"order":56,"depth":28,"number":7,"ownAudience":13,"audiences":57,"coverDeLight":7,"coverEnLight":7,"coverDeDark":7,"coverEnDark":7,"children":58},"_info/12_lizenzenunddrittanbieter","12_lizenzenunddrittanbieter","Licenses & Third-Party Notices","Automatically updated overview of the open-source components, licenses, and third-party notices.","/_info/licenses-and-third-party-notices/en","/_info/12_lizenzenunddrittanbieter/en",["Reactive",53],[51],"/_info/12_lizenzenunddrittanbieter","content/_Info/12_LizenzenUndDrittanbieter/en.md",12,[13],[],{"key":60,"slug":61,"title":62,"cleanTitle":62,"numberedTitle":62,"description":63,"path":64,"sourcePath":65,"aliases":66,"basePath":68,"filePath":69,"pageType":70,"hidden":10,"effectiveHidden":10,"order":71,"depth":28,"number":7,"ownAudience":13,"audiences":72,"coverDeLight":7,"coverEnLight":7,"coverDeDark":7,"coverEnDark":7,"children":73},"_info/13_mailinglisteundnewsletter","13_mailinglisteundnewsletter","Mailing List and Newsletter","Notifications about news and updates","/_info/mailing-list-and-newsletter/en","/_info/13_mailinglisteundnewsletter/en",["Reactive",67],[65],"/_info/13_mailinglisteundnewsletter","content/_Info/13_MailingListeUndNewsLetter/en.md","overview",13,[13],[74,94],{"key":75,"slug":76,"title":77,"cleanTitle":77,"numberedTitle":78,"description":79,"path":80,"sourcePath":81,"aliases":82,"basePath":84,"filePath":85,"pageType":70,"hidden":10,"effectiveHidden":10,"order":11,"depth":86,"number":87,"ownAudience":13,"audiences":88,"coverDeLight":89,"coverEnLight":90,"coverDeDark":91,"coverEnDark":92,"children":93},"_info/13_mailinglisteundnewsletter/newsletter","newsletter","Newsletter","1 Newsletter","Newsletter subscription","/_info/mailing-list-and-newsletter/newsletter/en","/_info/13_mailinglisteundnewsletter/newsletter/en",["Reactive",83],[81],"/_info/13_mailinglisteundnewsletter/newsletter","content/_Info/13_MailingListeUndNewsLetter/newsletter/en.md",3,"1",[13],"cover-de-light.png","cover-en-light.png","cover-de-dark.png","cover-en-dark.png",[],{"key":95,"slug":96,"title":97,"cleanTitle":97,"numberedTitle":98,"description":99,"path":100,"sourcePath":101,"aliases":102,"basePath":104,"filePath":105,"pageType":70,"hidden":10,"effectiveHidden":10,"order":28,"depth":86,"number":106,"ownAudience":13,"audiences":107,"coverDeLight":89,"coverEnLight":90,"coverDeDark":91,"coverEnDark":92,"children":108},"_info/13_mailinglisteundnewsletter/ifk","ifk","Mailing List for Computer Science Teachers in Cologne and the Surrounding Area","2 Mailing List for Computer Science Teachers in Cologne and the Surrounding Area","Discussion and exchange in the wider Cologne region","/_info/mailing-list-and-newsletter/cologne-teachers-mailing-list/en","/_info/13_mailinglisteundnewsletter/ifk/en",["Reactive",103],[101],"/_info/13_mailinglisteundnewsletter/ifk","content/_Info/13_MailingListeUndNewsLetter/ifk/en.md","2",[13],[],{"key":110,"slug":110,"title":111,"cleanTitle":111,"numberedTitle":111,"description":112,"path":113,"sourcePath":113,"aliases":114,"basePath":116,"filePath":117,"pageType":118,"hidden":10,"effectiveHidden":10,"order":-4,"depth":11,"number":7,"ownAudience":7,"audiences":119,"coverDeLight":89,"coverEnLight":90,"coverDeDark":91,"coverEnDark":92,"children":120},"_workshops","Workshops","Workshops and accompanying materials.","/_workshops/en",["Reactive",115],[],"/_workshops","content/_Workshops/en.md","chapter",[],[121],{"key":122,"slug":123,"title":124,"cleanTitle":124,"numberedTitle":125,"description":126,"path":127,"sourcePath":127,"aliases":128,"basePath":130,"filePath":131,"pageType":70,"hidden":10,"effectiveHidden":10,"order":132,"depth":28,"number":87,"ownAudience":7,"audiences":133,"coverDeLight":7,"coverEnLight":7,"coverDeDark":7,"coverEnDark":7,"children":134},"_workshops/2026_09_14","2026_09_14","Informatiktag20260914_AI-Tools","1 Informatiktag20260914_AI-Tools","Subpage for Informatiktag20260914_AI-Tools","/_workshops/2026_09_14/en",["Reactive",129],[],"/_workshops/2026_09_14","content/_Workshops/2026_09_14/en.md",2026,[],[135,150,169,184],{"key":136,"slug":137,"title":138,"cleanTitle":138,"numberedTitle":139,"description":140,"path":141,"sourcePath":142,"aliases":143,"basePath":145,"filePath":146,"pageType":26,"hidden":10,"effectiveHidden":10,"order":-4,"depth":86,"number":147,"ownAudience":7,"audiences":148,"coverDeLight":89,"coverEnLight":90,"coverDeDark":91,"coverEnDark":92,"children":149},"_workshops/2026_09_14/arbeitsauftrag1","arbeitsauftrag1","Task 1","1.1 Task 1","Explore and construct FNN decision regions using Heaviside activations.","/_workshops/2026_09_14/task-1/en","/_workshops/2026_09_14/arbeitsauftrag1/en",["Reactive",144],[142],"/_workshops/2026_09_14/arbeitsauftrag1","content/_Workshops/2026_09_14/arbeitsauftrag1/en.md","1.1",[],[],{"key":151,"slug":152,"title":153,"cleanTitle":153,"numberedTitle":154,"description":155,"path":156,"sourcePath":157,"aliases":158,"basePath":160,"filePath":161,"pageType":26,"hidden":10,"effectiveHidden":10,"order":-4,"depth":86,"number":162,"ownAudience":7,"audiences":163,"coverDeLight":164,"coverEnLight":165,"coverDeDark":166,"coverEnDark":167,"children":168},"_workshops/2026_09_14/arbeitsauftrag2","arbeitsauftrag2","Task 2","1.2 Task 2","Character classification with a Feedforward Neural Network.","/_workshops/2026_09_14/task-2/en","/_workshops/2026_09_14/arbeitsauftrag2/en",["Reactive",159],[157],"/_workshops/2026_09_14/arbeitsauftrag2","content/_Workshops/2026_09_14/arbeitsauftrag2/en.md","1.2",[],"cover-de-light.webp","cover-en-light.webp","cover-de-dark.webp","cover-en-dark.webp",[],{"key":170,"slug":171,"title":172,"cleanTitle":172,"numberedTitle":173,"description":174,"path":175,"sourcePath":176,"aliases":177,"basePath":179,"filePath":180,"pageType":26,"hidden":10,"effectiveHidden":10,"order":-4,"depth":86,"number":181,"ownAudience":7,"audiences":182,"coverDeLight":89,"coverEnLight":90,"coverDeDark":91,"coverEnDark":92,"children":183},"_workshops/2026_09_14/arbeitsauftrag3","arbeitsauftrag3","Task 3","1.3 Task 3","Experimental and mathematical comparison of CCE and MSE in FNN classification.","/_workshops/2026_09_14/task-3/en","/_workshops/2026_09_14/arbeitsauftrag3/en",["Reactive",178],[176],"/_workshops/2026_09_14/arbeitsauftrag3","content/_Workshops/2026_09_14/arbeitsauftrag3/en.md","1.3",[],[],{"key":185,"slug":186,"title":187,"cleanTitle":187,"numberedTitle":188,"description":189,"path":190,"sourcePath":191,"aliases":192,"basePath":194,"filePath":195,"pageType":26,"hidden":10,"effectiveHidden":10,"order":-4,"depth":86,"number":196,"ownAudience":7,"audiences":197,"coverDeLight":89,"coverEnLight":165,"coverDeDark":91,"coverEnDark":167,"children":198},"_workshops/2026_09_14/arbeitsauftrag4","arbeitsauftrag4","Task 4","1.4 Task 4","Two fundamental ideas of RL using bandit problems.","/_workshops/2026_09_14/task-4/en","/_workshops/2026_09_14/arbeitsauftrag4/en",["Reactive",193],[191],"/_workshops/2026_09_14/arbeitsauftrag4","content/_Workshops/2026_09_14/arbeitsauftrag4/en.md","1.4",[],[],{"_path":191,"_dir":186,"_draft":10,"_partial":200,"_locale":7,"title":187,"description":189,"navigation":200,"author":201,"pageType":26,"coverDeLight":89,"coverEnLight":165,"coverDeDark":91,"coverEnDark":167,"pageId":202,"slug":203,"aliases":204,"body":205,"_type":883,"_id":884,"_source":26,"_file":885,"_stem":886,"_extension":887},true,"HM_JDS","070d0bec-aa3c-48d6-a5ac-ea34c4f6aa7d","task-4",[191],{"type":206,"children":207,"toc":871},"root",[208,215,229,236,249,260,279,290,302,307,327,333,338,405,412,417,422,434,439,444,495,506,515,593,599,611,622,626,631,638,643,647,652,657,663,668,672,683,689,695,712,716,721,725,731,736,745,750,755,759,764,768,779,785,790,795,799,804,808,813,817,822,827],{"type":209,"tag":210,"props":211,"children":212},"element","h1",{"id":203},[213],{"type":214,"value":187},"text",{"type":209,"tag":216,"props":217,"children":218},"p",{},[219,221,227],{"type":214,"value":220},"In this assignment, you will investigate fundamental approaches for dealing with the ",{"type":209,"tag":222,"props":223,"children":224},"strong",{},[225],{"type":214,"value":226},"exploration-exploitation trade-off",{"type":214,"value":228}," in bandit problems and compare different ways of updating value estimates.",{"type":209,"tag":230,"props":231,"children":233},"h2",{"id":232},"exercise-1-exploration-and-exploitation",[234],{"type":214,"value":235},"Exercise 1 – Exploration and Exploitation",{"type":209,"tag":216,"props":237,"children":238},{},[239,241,247],{"type":214,"value":240},"Open the ",{"type":209,"tag":242,"props":243,"children":244},"em",{},[245],{"type":214,"value":246},"Exploration vs. Exploitation",{"type":214,"value":248}," app.",{"type":209,"tag":216,"props":250,"children":251},{},[252],{"type":209,"tag":253,"props":254,"children":257},"a",{"href":255,"style":256},"/tools/KI_ML/03_ReinforcementLearning/04_bandit_algorithmen/01_exploration_vs_exploitation","font-size: 1.4em; font-weight: 700;",[258],{"type":214,"value":259},"Open the Exploration vs. Exploitation tool",{"type":209,"tag":216,"props":261,"children":262},{},[263,265,270,272,277],{"type":214,"value":264},"Choose one of the two contexts ",{"type":209,"tag":242,"props":266,"children":267},{},[268],{"type":214,"value":269},"Party Opener",{"type":214,"value":271}," or ",{"type":209,"tag":242,"props":273,"children":274},{},[275],{"type":214,"value":276},"Way to School",{"type":214,"value":278},".",{"type":209,"tag":216,"props":280,"children":281},{},[282,284,289],{"type":214,"value":283},"First, try to maximize the sum of the rewards you receive by making your own decisions. Use the game mode ",{"type":209,"tag":242,"props":285,"children":286},{},[287],{"type":214,"value":288},"I decide myself",{"type":214,"value":278},{"type":209,"tag":216,"props":291,"children":292},{},[293,295,300],{"type":214,"value":294},"Then compare your results with those of an agent in ",{"type":209,"tag":242,"props":296,"children":297},{},[298],{"type":214,"value":299},"Auto",{"type":214,"value":301}," mode.",{"type":209,"tag":216,"props":303,"children":304},{},[305],{"type":214,"value":306},"In particular, investigate:",{"type":209,"tag":308,"props":309,"children":310},"ul",{},[311,317,322],{"type":209,"tag":312,"props":313,"children":314},"li",{},[315],{"type":214,"value":316},"What happens if the agent does not explore at all or explores only very little?",{"type":209,"tag":312,"props":318,"children":319},{},[320],{"type":214,"value":321},"What happens if the agent explores very frequently?",{"type":209,"tag":312,"props":323,"children":324},{},[325],{"type":214,"value":326},"Why can neither exploring exclusively nor always exploiting the currently best action be optimal in the long run?",{"type":209,"tag":230,"props":328,"children":330},{"id":329},"exercise-2-methods-for-ensuring-exploration",[331],{"type":214,"value":332},"Exercise 2 – Methods for Ensuring Exploration",{"type":209,"tag":216,"props":334,"children":335},{},[336],{"type":214,"value":337},"Computer science teacher Ms Hempel is marking the tests of three introductory computer science courses (GK 1, GK 2 and GK 3).",{"type":209,"tag":339,"props":340,"children":341},"table",{},[342,361],{"type":209,"tag":343,"props":344,"children":345},"thead",{},[346],{"type":209,"tag":347,"props":348,"children":349},"tr",{},[350,356],{"type":209,"tag":351,"props":352,"children":353},"th",{},[354],{"type":214,"value":355},"Course (Action)",{"type":209,"tag":351,"props":357,"children":358},{},[359],{"type":214,"value":360},"Grades (Rewards)",{"type":209,"tag":362,"props":363,"children":364},"tbody",{},[365,379,392],{"type":209,"tag":347,"props":366,"children":367},{},[368,374],{"type":209,"tag":369,"props":370,"children":371},"td",{},[372],{"type":214,"value":373},"GK 1",{"type":209,"tag":369,"props":375,"children":376},{},[377],{"type":214,"value":378},"4, 2, 3, 3, 5, 2, 1, 1, 4",{"type":209,"tag":347,"props":380,"children":381},{},[382,387],{"type":209,"tag":369,"props":383,"children":384},{},[385],{"type":214,"value":386},"GK 2",{"type":209,"tag":369,"props":388,"children":389},{},[390],{"type":214,"value":391},"3, 4, 4, 1, 1, 1, 2, 5",{"type":209,"tag":347,"props":393,"children":394},{},[395,400],{"type":209,"tag":369,"props":396,"children":397},{},[398],{"type":214,"value":399},"GK 3",{"type":209,"tag":369,"props":401,"children":402},{},[403],{"type":214,"value":404},"2, 5, 2, 5, 2, 5, 1, 6",{"type":209,"tag":406,"props":407,"children":409},"h3",{"id":408},"a-optimistic-initial-values",[410],{"type":214,"value":411},"a) Optimistic Initial Values",{"type":209,"tag":216,"props":413,"children":414},{},[415],{"type":214,"value":416},"Ms Hempel is initially “optimistic” and assigns all three courses the initial value",{"type":209,"tag":418,"props":419,"children":421},"d-ma",{"code":420},"Q_1(a)=6.",[],{"type":209,"tag":216,"props":423,"children":424},{},[425,427,432],{"type":214,"value":426},"She uses a ",{"type":209,"tag":222,"props":428,"children":429},{},[430],{"type":214,"value":431},"greedy strategy with optimistic initial values",{"type":214,"value":433},". In the event of a tie, she always chooses a test from the course with the lower course number.",{"type":209,"tag":216,"props":435,"children":436},{},[437],{"type":214,"value":438},"Determine the order in which she marks the first seven tests in each case.",{"type":209,"tag":216,"props":440,"children":441},{},[442],{"type":214,"value":443},"Investigate the following three variants:",{"type":209,"tag":445,"props":446,"children":447},"ol",{},[448,465,480],{"type":209,"tag":312,"props":449,"children":450},{},[451,456,458,463],{"type":209,"tag":222,"props":452,"children":453},{},[454],{"type":214,"value":455},"Sample average:",{"type":214,"value":457}," The initial value is ",{"type":209,"tag":459,"props":460,"children":462},"i-ma",{"code":461},"Q_1(a)=6",[],{"type":214,"value":464},", but it does not count as an observation.",{"type":209,"tag":312,"props":466,"children":467},{},[468,473,475,479],{"type":209,"tag":222,"props":469,"children":470},{},[471],{"type":214,"value":472},"Constant step size:",{"type":214,"value":474}," Use ",{"type":209,"tag":459,"props":476,"children":478},{"code":477},"\\alpha=0.5",[],{"type":214,"value":278},{"type":209,"tag":312,"props":481,"children":482},{},[483,488,490,493],{"type":209,"tag":222,"props":484,"children":485},{},[486],{"type":214,"value":487},"Pseudo-observation:",{"type":214,"value":489}," The initial value ",{"type":209,"tag":459,"props":491,"children":492},{"code":461},[],{"type":214,"value":494}," is treated as an additional observation when calculating the mean.",{"type":209,"tag":216,"props":496,"children":497},{},[498,500,505],{"type":214,"value":499},"Check your solutions using the ",{"type":209,"tag":242,"props":501,"children":502},{},[503],{"type":214,"value":504},"Bandit Exploration Strategies",{"type":214,"value":248},{"type":209,"tag":216,"props":507,"children":508},{},[509],{"type":209,"tag":253,"props":510,"children":512},{"href":511,"style":256},"/tools/KI_ML/03_ReinforcementLearning/04_bandit_algorithmen/02_bandit_exploration_strategien",[513],{"type":214,"value":514},"Open Bandit Exploration Strategies",{"type":209,"tag":516,"props":517,"children":520},"info-box",{"label":518,"title":519},"App: Bandit Exploration Strategies","Check your solution",[521],{"type":209,"tag":445,"props":522,"children":523},{},[524,537,551,562,574,583,588],{"type":209,"tag":312,"props":525,"children":526},{},[527,529,536],{"type":214,"value":528},"Download the ",{"type":209,"tag":530,"props":531,"children":533},"content-asset-link",{"src":532,"download":200},"media/klassenarbeiten-bandit-beispiel-v4.json",[534],{"type":214,"value":535},"JSON file containing the example data",{"type":214,"value":278},{"type":209,"tag":312,"props":538,"children":539},{},[540,542,549],{"type":214,"value":541},"Load the file ",{"type":209,"tag":543,"props":544,"children":546},"code",{"className":545},[],[547],{"type":214,"value":548},"klassenarbeiten-bandit-beispiel-v4.json",{"type":214,"value":550}," in the tool.",{"type":209,"tag":312,"props":552,"children":553},{},[554,555,560],{"type":214,"value":240},{"type":209,"tag":242,"props":556,"children":557},{},[558],{"type":214,"value":559},"Optimistic Initial Values",{"type":214,"value":561}," tab.",{"type":209,"tag":312,"props":563,"children":564},{},[565,567,572],{"type":214,"value":566},"Under ",{"type":209,"tag":242,"props":568,"children":569},{},[570],{"type":214,"value":571},"Options",{"type":214,"value":573},", select the variant you want to investigate.",{"type":209,"tag":312,"props":575,"children":576},{},[577,579,582],{"type":214,"value":578},"For the variant with a constant step size, use ",{"type":209,"tag":459,"props":580,"children":581},{"code":477},[],{"type":214,"value":278},{"type":209,"tag":312,"props":584,"children":585},{},[586],{"type":214,"value":587},"Execute the first steps one at a time and compare the calculations shown with your own solution.",{"type":209,"tag":312,"props":589,"children":590},{},[591],{"type":214,"value":592},"Then reset the tab and investigate the next variant.",{"type":209,"tag":406,"props":594,"children":596},{"id":595},"b-upper-confidence-bound",[597],{"type":214,"value":598},"b) Upper Confidence Bound",{"type":209,"tag":216,"props":600,"children":601},{},[602,604,609],{"type":214,"value":603},"Ms Hempel now uses an ",{"type":209,"tag":222,"props":605,"children":606},{},[607],{"type":214,"value":608},"Upper Confidence Bound method",{"type":214,"value":610}," to select the next test to mark.",{"type":209,"tag":216,"props":612,"children":613},{},[614,616,620],{"type":214,"value":615},"First, she marks one test from each course. To select the fourth test (",{"type":209,"tag":459,"props":617,"children":619},{"code":618},"t=4",[],{"type":214,"value":621},"), the value",{"type":209,"tag":418,"props":623,"children":625},{"code":624},"U_t(a)\n=\nQ_t(a)\n+\nc\\cdot\n\\sqrt{\n\\frac{\\log_2(t)}{N_t(a)}\n}",[],{"type":209,"tag":216,"props":627,"children":628},{},[629],{"type":214,"value":630},"is then calculated for each action.",{"type":209,"tag":632,"props":633,"children":635},"h4",{"id":634},"i-selecting-the-fourth-test",[636],{"type":214,"value":637},"i) Selecting the Fourth Test",{"type":209,"tag":216,"props":639,"children":640},{},[641],{"type":214,"value":642},"Explain, without carrying out a detailed calculation, why",{"type":209,"tag":418,"props":644,"children":646},{"code":645},"U_4(\\mathrm{GK}\\,1) > U_4(\\mathrm{GK}\\,2) > U_4(\\mathrm{GK}\\,3)",[],{"type":209,"tag":216,"props":648,"children":649},{},[650],{"type":214,"value":651},"holds.",{"type":209,"tag":216,"props":653,"children":654},{},[655],{"type":214,"value":656},"Which test is therefore marked fourth?",{"type":209,"tag":632,"props":658,"children":660},{"id":659},"ii-selecting-the-fifth-test",[661],{"type":214,"value":662},"ii) Selecting the Fifth Test",{"type":209,"tag":216,"props":664,"children":665},{},[666],{"type":214,"value":667},"The fourth test marked is the second test from GK 1. Afterwards, the estimated action values satisfy",{"type":209,"tag":418,"props":669,"children":671},{"code":670},"Q(\\mathrm{GK}\\,1) = Q(\\mathrm{GK}\\,2) > Q(\\mathrm{GK}\\,3).",[],{"type":209,"tag":216,"props":673,"children":674},{},[675,677,681],{"type":214,"value":676},"Using the selection counts ",{"type":209,"tag":459,"props":678,"children":680},{"code":679},"N_t(a)",[],{"type":214,"value":682}," and the exploration term in the UCB formula, explain why a test from GK 2 is selected next.",{"type":209,"tag":230,"props":684,"children":686},{"id":685},"exercise-3-updating-knowledge-incrementally",[687],{"type":214,"value":688},"Exercise 3 – Updating Knowledge Incrementally",{"type":209,"tag":406,"props":690,"children":692},{"id":691},"a-calculating-the-arithmetic-mean-incrementally",[693],{"type":214,"value":694},"a) Calculating the Arithmetic Mean Incrementally",{"type":209,"tag":216,"props":696,"children":697},{},[698,700,704,706,710],{"type":214,"value":699},"Show by algebraic transformation that for the arithmetic mean ",{"type":209,"tag":459,"props":701,"children":703},{"code":702},"\\bar{x}_{n+1}",[],{"type":214,"value":705}," of ",{"type":209,"tag":459,"props":707,"children":709},{"code":708},"n+1",[],{"type":214,"value":711}," numbers",{"type":209,"tag":418,"props":713,"children":715},{"code":714},"x_1,\\ldots,x_n,x_{n+1}",[],{"type":209,"tag":216,"props":717,"children":718},{},[719],{"type":214,"value":720},"the following equation holds:",{"type":209,"tag":418,"props":722,"children":724},{"code":723},"\\bar{x}_{n+1}\n=\n\\bar{x}_n\n+\n\\frac{1}{n+1}\n\\left(\nx_{n+1}-\\bar{x}_n\n\\right).",[],{"type":209,"tag":406,"props":726,"children":728},{"id":727},"b-visualizing-incremental-updates",[729],{"type":214,"value":730},"b) Visualizing Incremental Updates",{"type":209,"tag":216,"props":732,"children":733},{},[734],{"type":214,"value":735},"Open the app for incrementally calculating the arithmetic mean.",{"type":209,"tag":216,"props":737,"children":738},{},[739],{"type":209,"tag":253,"props":740,"children":742},{"href":741,"style":256},"/tools/KI_ML/03_ReinforcementLearning/04_bandit_algorithmen/03_Inkrementelles_Mittel",[743],{"type":214,"value":744},"Open the Incremental Calculation of the Arithmetic Mean tool",{"type":209,"tag":216,"props":746,"children":747},{},[748],{"type":214,"value":749},"Using a suitable example, investigate how the previous mean changes after each new observation.",{"type":209,"tag":216,"props":751,"children":752},{},[753],{"type":214,"value":754},"Pay particular attention to the correction term",{"type":209,"tag":418,"props":756,"children":758},{"code":757},"x_{n+1}-\\bar{x}_n",[],{"type":209,"tag":216,"props":760,"children":761},{},[762],{"type":214,"value":763},"and to the factor",{"type":209,"tag":418,"props":765,"children":767},{"code":766},"\\frac{1}{n+1}.",[],{"type":209,"tag":216,"props":769,"children":770},{},[771,773,777],{"type":214,"value":772},"Describe how the influence of a single new observation changes as ",{"type":209,"tag":459,"props":774,"children":776},{"code":775},"n",[],{"type":214,"value":778}," increases.",{"type":209,"tag":406,"props":780,"children":782},{"id":781},"c-comparison-with-q-learning",[783],{"type":214,"value":784},"c) Comparison with Q-Learning",{"type":209,"tag":216,"props":786,"children":787},{},[788],{"type":214,"value":789},"In many value-based reinforcement learning methods, previous value estimates are updated using a new observation.",{"type":209,"tag":216,"props":791,"children":792},{},[793],{"type":214,"value":794},"In tabular Q-learning, the update rule can be written as",{"type":209,"tag":418,"props":796,"children":798},{"code":797},"Q'(s,a)\n=\n(1-\\alpha)\\cdot Q(s,a)\n+\n\\alpha\\cdot\n\\left(\nR(s,a)\n+\n\\gamma\\cdot\\max_{a'}Q(s',a')\n\\right).\n\\tag{1}",[],{"type":209,"tag":216,"props":800,"children":801},{},[802],{"type":214,"value":803},"Equivalently,",{"type":209,"tag":418,"props":805,"children":807},{"code":806},"Q'(s,a)\n=\nQ(s,a)\n+\n\\alpha\\cdot\n\\left(\nR(s,a)\n+\n\\gamma\\cdot\\max_{a'}Q(s',a')\n-\nQ(s,a)\n\\right).\n\\tag{2}",[],{"type":209,"tag":216,"props":809,"children":810},{},[811],{"type":214,"value":812},"The update of the arithmetic mean can also be written in a form similar to Equation (1):",{"type":209,"tag":418,"props":814,"children":816},{"code":815},"\\bar{x}_{n+1}\n=\n\\frac{n}{n+1}\\bar{x}_n\n+\n\\frac{1}{n+1}x_{n+1}.",[],{"type":209,"tag":216,"props":818,"children":819},{},[820],{"type":214,"value":821},"Compare the update rules.",{"type":209,"tag":216,"props":823,"children":824},{},[825],{"type":214,"value":826},"In particular, address the following questions:",{"type":209,"tag":308,"props":828,"children":829},{},[830,835,840,845,862],{"type":209,"tag":312,"props":831,"children":832},{},[833],{"type":214,"value":834},"Which quantity represents the previous knowledge in each case?",{"type":209,"tag":312,"props":836,"children":837},{},[838],{"type":214,"value":839},"Which quantity contains the new information?",{"type":209,"tag":312,"props":841,"children":842},{},[843],{"type":214,"value":844},"What role does the weighting factor play?",{"type":209,"tag":312,"props":846,"children":847},{},[848,850,854,856,860],{"type":214,"value":849},"How does the factor ",{"type":209,"tag":459,"props":851,"children":853},{"code":852},"\\frac{1}{n+1}",[],{"type":214,"value":855}," differ from a constant step size ",{"type":209,"tag":459,"props":857,"children":859},{"code":858},"\\alpha",[],{"type":214,"value":861},"?",{"type":209,"tag":312,"props":863,"children":864},{},[865,867],{"type":214,"value":866},"What additional information is contained in the Q-learning target",{"type":209,"tag":418,"props":868,"children":870},{"code":869},"R(s,a)+\\gamma\\max_{a'}Q(s',a')?",[],{"title":7,"searchDepth":28,"depth":28,"links":872},[873,874,878],{"id":232,"depth":28,"text":235},{"id":329,"depth":28,"text":332,"children":875},[876,877],{"id":408,"depth":86,"text":411},{"id":595,"depth":86,"text":598},{"id":685,"depth":28,"text":688,"children":879},[880,881,882],{"id":691,"depth":86,"text":694},{"id":727,"depth":86,"text":730},{"id":781,"depth":86,"text":784},"markdown","content:_Workshops:2026_09_14:arbeitsauftrag4:en.md","_Workshops/2026_09_14/arbeitsauftrag4/en.md","_Workshops/2026_09_14/arbeitsauftrag4/en","md",1793367297185]