[ { "startMs": 1309, "text": "[music]" }, { "startMs": 12639, "text": "All right. Hello everybody. Thank you so" }, { "startMs": 14230, "text": "All right. Hello everybody. Thank you so" }, { "startMs": 14240, "text": "All right. Hello everybody. Thank you so much for coming. My name is Hassan. Uh" }, { "startMs": 16230, "text": "much for coming. My name is Hassan. Uh" }, { "startMs": 16240, "text": "much for coming. My name is Hassan. Uh this is my dad Omar. And uh we're very" }, { "startMs": 18550, "text": "this is my dad Omar. And uh we're very" }, { "startMs": 18560, "text": "this is my dad Omar. And uh we're very excited to be here today uh to to talk." }, { "startMs": 21109, "text": "excited to be here today uh to to talk." }, { "startMs": 21119, "text": "excited to be here today uh to to talk. Uh I think the only father-son talk at" }, { "startMs": 23590, "text": "Uh I think the only father-son talk at" }, { "startMs": 23600, "text": "Uh I think the only father-son talk at this conference. So um thank you." }, { "startMs": 26190, "text": "this conference. So um thank you." }, { "startMs": 26200, "text": "this conference. So um thank you. [cheering] Thank you. [applause]" }, { "startMs": 26870, "text": "[cheering] Thank you. [applause]" }, { "startMs": 26880, "text": "[cheering] Thank you. [applause] Thanks." }, { "startMs": 29920, "text": "So, I'm here to talk about, you know," }, { "startMs": 31750, "text": "So, I'm here to talk about, you know," }, { "startMs": 31760, "text": "So, I'm here to talk about, you know, can LLMs actually uh make financial" }, { "startMs": 35190, "text": "can LLMs actually uh make financial" }, { "startMs": 35200, "text": "can LLMs actually uh make financial forecasts of a company? And so, we used" }, { "startMs": 36950, "text": "forecasts of a company? And so, we used" }, { "startMs": 36960, "text": "forecasts of a company? And so, we used Costco. We gave an open model kind of" }, { "startMs": 39350, "text": "Costco. We gave an open model kind of" }, { "startMs": 39360, "text": "Costco. We gave an open model kind of everything uh that was public from" }, { "startMs": 41350, "text": "everything uh that was public from" }, { "startMs": 41360, "text": "everything uh that was public from Costco's filings up until a certain" }, { "startMs": 43030, "text": "Costco's filings up until a certain" }, { "startMs": 43040, "text": "Costco's filings up until a certain point, and we had it try to predict" }, { "startMs": 44950, "text": "point, and we had it try to predict" }, { "startMs": 44960, "text": "point, and we had it try to predict Costco's profits over the next eight" }, { "startMs": 47350, "text": "Costco's profits over the next eight" }, { "startMs": 47360, "text": "Costco's profits over the next eight quarters. So, we'll start off with some" }, { "startMs": 49350, "text": "quarters. So, we'll start off with some" }, { "startMs": 49360, "text": "quarters. So, we'll start off with some intros. My name is Hassan. Uh I lead" }, { "startMs": 51430, "text": "intros. My name is Hassan. Uh I lead" }, { "startMs": 51440, "text": "intros. My name is Hassan. Uh I lead developer experience at Together AI. The" }, { "startMs": 53110, "text": "developer experience at Together AI. The" }, { "startMs": 53120, "text": "developer experience at Together AI. The main thing to know about me is I really" }, { "startMs": 54470, "text": "main thing to know about me is I really" }, { "startMs": 54480, "text": "main thing to know about me is I really like building AI apps. Everything I do" }, { "startMs": 56229, "text": "like building AI apps. Everything I do" }, { "startMs": 56239, "text": "like building AI apps. Everything I do is open source on GitHub. And uh this is" }, { "startMs": 58310, "text": "is open source on GitHub. And uh this is" }, { "startMs": 58320, "text": "is open source on GitHub. And uh this is another one of the apps that I built uh" }, { "startMs": 59990, "text": "another one of the apps that I built uh" }, { "startMs": 60000, "text": "another one of the apps that I built uh but this one was in collaboration uh" }, { "startMs": 61750, "text": "but this one was in collaboration uh" }, { "startMs": 61760, "text": "but this one was in collaboration uh with my dad and I work over at Together" }, { "startMs": 63990, "text": "with my dad and I work over at Together" }, { "startMs": 64000, "text": "with my dad and I work over at Together AAI. We used it for the open source" }, { "startMs": 65750, "text": "AAI. We used it for the open source" }, { "startMs": 65760, "text": "AAI. We used it for the open source models in in this talk. We're a platform" }, { "startMs": 67670, "text": "models in in this talk. We're a platform" }, { "startMs": 67680, "text": "models in in this talk. We're a platform for uh running and deploying and" }, { "startMs": 70469, "text": "for uh running and deploying and" }, { "startMs": 70479, "text": "for uh running and deploying and training open source models." }, { "startMs": 72950, "text": "training open source models." }, { "startMs": 72960, "text": "training open source models. Hey guys, my name is Omargari. So um" }, { "startMs": 75990, "text": "Hey guys, my name is Omargari. So um" }, { "startMs": 76000, "text": "Hey guys, my name is Omargari. So um usually parents they teach children but" }, { "startMs": 78149, "text": "usually parents they teach children but" }, { "startMs": 78159, "text": "usually parents they teach children but Hessen he was teaching me a lot during" }, { "startMs": 80550, "text": "Hessen he was teaching me a lot during" }, { "startMs": 80560, "text": "Hessen he was teaching me a lot during the last six years when I uh I got to" }, { "startMs": 82710, "text": "the last six years when I uh I got to" }, { "startMs": 82720, "text": "the last six years when I uh I got to coding after 25 years of experience with" }, { "startMs": 85270, "text": "coding after 25 years of experience with" }, { "startMs": 85280, "text": "coding after 25 years of experience with the big four firms auditing firms uh" }, { "startMs": 87990, "text": "the big four firms auditing firms uh" }, { "startMs": 88000, "text": "the big four firms auditing firms uh couple of positions as a CFO and audit" }, { "startMs": 90469, "text": "couple of positions as a CFO and audit" }, { "startMs": 90479, "text": "couple of positions as a CFO and audit uh head of internal audit of" }, { "startMs": 92390, "text": "uh head of internal audit of" }, { "startMs": 92400, "text": "uh head of internal audit of multi-billion dollar company. So today" }, { "startMs": 95190, "text": "multi-billion dollar company. So today" }, { "startMs": 95200, "text": "multi-billion dollar company. So today I'm in the human in the loop for" }, { "startMs": 96950, "text": "I'm in the human in the loop for" }, { "startMs": 96960, "text": "I'm in the human in the loop for Hassan's um engineering work." }, { "startMs": 101350, "text": "Hassan's um engineering work." }, { "startMs": 101360, "text": "Hassan's um engineering work. Okay. So my coding experience didn't" }, { "startMs": 104310, "text": "Okay. So my coding experience didn't" }, { "startMs": 104320, "text": "Okay. So my coding experience didn't start just last six years. Um I had a" }, { "startMs": 106789, "text": "start just last six years. Um I had a" }, { "startMs": 106799, "text": "start just last six years. Um I had a lot of passion for coding at at age of" }, { "startMs": 108630, "text": "lot of passion for coding at at age of" }, { "startMs": 108640, "text": "lot of passion for coding at at age of 16. I was coding in assembly and the" }, { "startMs": 111510, "text": "16. I was coding in assembly and the" }, { "startMs": 111520, "text": "16. I was coding in assembly and the reason I did that I started with basic a" }, { "startMs": 113749, "text": "reason I did that I started with basic a" }, { "startMs": 113759, "text": "reason I did that I started with basic a tennis game small tennis game on an" }, { "startMs": 115590, "text": "tennis game small tennis game on an" }, { "startMs": 115600, "text": "tennis game small tennis game on an epson with four lines on the screen and" }, { "startMs": 118550, "text": "epson with four lines on the screen and" }, { "startMs": 118560, "text": "epson with four lines on the screen and just this four um you know tennis uh" }, { "startMs": 120709, "text": "just this four um you know tennis uh" }, { "startMs": 120719, "text": "just this four um you know tennis uh court but the ball was too slow. It took" }, { "startMs": 123270, "text": "court but the ball was too slow. It took" }, { "startMs": 123280, "text": "court but the ball was too slow. It took like almost a minute to get to the other" }, { "startMs": 125030, "text": "like almost a minute to get to the other" }, { "startMs": 125040, "text": "like almost a minute to get to the other side. And then I learned assembly" }, { "startMs": 127030, "text": "side. And then I learned assembly" }, { "startMs": 127040, "text": "side. And then I learned assembly because of my brother's uh IT" }, { "startMs": 128630, "text": "because of my brother's uh IT" }, { "startMs": 128640, "text": "because of my brother's uh IT background. And then I rewrote the the" }, { "startMs": 131270, "text": "background. And then I rewrote the the" }, { "startMs": 131280, "text": "background. And then I rewrote the the same game and it took the ball. I can't" }, { "startMs": 133350, "text": "same game and it took the ball. I can't" }, { "startMs": 133360, "text": "same game and it took the ball. I can't see it anymore. It was too fast to so I" }, { "startMs": 136309, "text": "see it anymore. It was too fast to so I" }, { "startMs": 136319, "text": "see it anymore. It was too fast to so I had to slow it down to really see" }, { "startMs": 138070, "text": "had to slow it down to really see" }, { "startMs": 138080, "text": "had to slow it down to really see things. So just wanted to share that" }, { "startMs": 139510, "text": "things. So just wanted to share that" }, { "startMs": 139520, "text": "things. So just wanted to share that with you." }, { "startMs": 141270, "text": "with you." }, { "startMs": 141280, "text": "with you. All right. So spoiler alert. Um the LLM" }, { "startMs": 145030, "text": "All right. So spoiler alert. Um the LLM" }, { "startMs": 145040, "text": "All right. So spoiler alert. Um the LLM was accurate up until about 4.4%. So the" }, { "startMs": 148550, "text": "was accurate up until about 4.4%. So the" }, { "startMs": 148560, "text": "was accurate up until about 4.4%. So the our our forecast of Costco's actual" }, { "startMs": 150550, "text": "our our forecast of Costco's actual" }, { "startMs": 150560, "text": "our our forecast of Costco's actual membership was off by 4.4%. And so I I" }, { "startMs": 152869, "text": "membership was off by 4.4%. And so I I" }, { "startMs": 152879, "text": "membership was off by 4.4%. And so I I thought this was pretty good, right? We" }, { "startMs": 154229, "text": "thought this was pretty good, right? We" }, { "startMs": 154239, "text": "thought this was pretty good, right? We had this LLM build a financial model. We" }, { "startMs": 156150, "text": "had this LLM build a financial model. We" }, { "startMs": 156160, "text": "had this LLM build a financial model. We evaled it. Um ended up giving us this" }, { "startMs": 158229, "text": "evaled it. Um ended up giving us this" }, { "startMs": 158239, "text": "evaled it. Um ended up giving us this result. Um, and then I showed it to my" }, { "startMs": 160630, "text": "result. Um, and then I showed it to my" }, { "startMs": 160640, "text": "result. Um, and then I showed it to my dad and it turned out it got the right" }, { "startMs": 162710, "text": "dad and it turned out it got the right" }, { "startMs": 162720, "text": "dad and it turned out it got the right answer or got close to the right answer," }, { "startMs": 164630, "text": "answer or got close to the right answer," }, { "startMs": 164640, "text": "answer or got close to the right answer, uh, but for the wrong reasons. And so" }, { "startMs": 165990, "text": "uh, but for the wrong reasons. And so" }, { "startMs": 166000, "text": "uh, but for the wrong reasons. And so the rest of this talk is going to be" }, { "startMs": 167270, "text": "the rest of this talk is going to be" }, { "startMs": 167280, "text": "the rest of this talk is going to be going over, you know, how we built this" }, { "startMs": 168869, "text": "going over, you know, how we built this" }, { "startMs": 168879, "text": "going over, you know, how we built this financial model, how we evalued it, and" }, { "startMs": 170949, "text": "financial model, how we evalued it, and" }, { "startMs": 170959, "text": "financial model, how we evalued it, and actually what it ended up getting wrong" }, { "startMs": 172790, "text": "actually what it ended up getting wrong" }, { "startMs": 172800, "text": "actually what it ended up getting wrong and and kind of lessons learned along" }, { "startMs": 174550, "text": "and and kind of lessons learned along" }, { "startMs": 174560, "text": "and and kind of lessons learned along the way, including kind of the" }, { "startMs": 175750, "text": "the way, including kind of the" }, { "startMs": 175760, "text": "the way, including kind of the overarching lesson, which is having like" }, { "startMs": 177910, "text": "overarching lesson, which is having like" }, { "startMs": 177920, "text": "overarching lesson, which is having like an expert human in the loop, is is very," }, { "startMs": 179830, "text": "an expert human in the loop, is is very," }, { "startMs": 179840, "text": "an expert human in the loop, is is very, very important when you're building, you" }, { "startMs": 181430, "text": "very important when you're building, you" }, { "startMs": 181440, "text": "very important when you're building, you know, AI agents in in these different" }, { "startMs": 183030, "text": "know, AI agents in in these different" }, { "startMs": 183040, "text": "know, AI agents in in these different spaces. So, really quick background" }, { "startMs": 185350, "text": "spaces. So, really quick background" }, { "startMs": 185360, "text": "spaces. So, really quick background about Costco. Costco sells a lot of" }, { "startMs": 187750, "text": "about Costco. Costco sells a lot of" }, { "startMs": 187760, "text": "about Costco. Costco sells a lot of stuff. They sell $300 billion worth of" }, { "startMs": 189589, "text": "stuff. They sell $300 billion worth of" }, { "startMs": 189599, "text": "stuff. They sell $300 billion worth of stuff every year at close to cost. So" }, { "startMs": 191910, "text": "stuff every year at close to cost. So" }, { "startMs": 191920, "text": "stuff every year at close to cost. So they make very little money from that" }, { "startMs": 193430, "text": "they make very little money from that" }, { "startMs": 193440, "text": "they make very little money from that $300 billion. Um actually more than half" }, { "startMs": 195830, "text": "$300 billion. Um actually more than half" }, { "startMs": 195840, "text": "$300 billion. Um actually more than half of their revenue is from their" }, { "startMs": 197110, "text": "of their revenue is from their" }, { "startMs": 197120, "text": "of their revenue is from their membership fees, right? So they charge" }, { "startMs": 198470, "text": "membership fees, right? So they charge" }, { "startMs": 198480, "text": "membership fees, right? So they charge you to to come at the door. Um their" }, { "startMs": 201430, "text": "you to to come at the door. Um their" }, { "startMs": 201440, "text": "you to to come at the door. Um their total operating profit was about $12" }, { "startMs": 203270, "text": "total operating profit was about $12" }, { "startMs": 203280, "text": "total operating profit was about $12 billion and and a little bit more than" }, { "startMs": 204869, "text": "billion and and a little bit more than" }, { "startMs": 204879, "text": "billion and and a little bit more than half of that was from membership fees." }, { "startMs": 206550, "text": "half of that was from membership fees." }, { "startMs": 206560, "text": "half of that was from membership fees. So membership fees are are, you know," }, { "startMs": 208070, "text": "So membership fees are are, you know," }, { "startMs": 208080, "text": "So membership fees are are, you know, extremely important when we're trying to" }, { "startMs": 209509, "text": "extremely important when we're trying to" }, { "startMs": 209519, "text": "extremely important when we're trying to forecast uh Costco's uh revenue and" }, { "startMs": 212070, "text": "forecast uh Costco's uh revenue and" }, { "startMs": 212080, "text": "forecast uh Costco's uh revenue and profits. And thankfully Costco gave us" }, { "startMs": 214470, "text": "profits. And thankfully Costco gave us" }, { "startMs": 214480, "text": "profits. And thankfully Costco gave us the perfect test on July 2024 because" }, { "startMs": 217190, "text": "the perfect test on July 2024 because" }, { "startMs": 217200, "text": "the perfect test on July 2024 because they raised their membership prices for" }, { "startMs": 219670, "text": "they raised their membership prices for" }, { "startMs": 219680, "text": "they raised their membership prices for the first time since 2017. So Costco has" }, { "startMs": 222470, "text": "the first time since 2017. So Costco has" }, { "startMs": 222480, "text": "the first time since 2017. So Costco has two tiers. They have a standard tier" }, { "startMs": 223670, "text": "two tiers. They have a standard tier" }, { "startMs": 223680, "text": "two tiers. They have a standard tier that they raised from 60 to $65 and they" }, { "startMs": 226390, "text": "that they raised from 60 to $65 and they" }, { "startMs": 226400, "text": "that they raised from 60 to $65 and they have an executive tier that they raised" }, { "startMs": 227990, "text": "have an executive tier that they raised" }, { "startMs": 228000, "text": "have an executive tier that they raised from 120 to $130 uh. So what we did is" }, { "startMs": 231509, "text": "from 120 to $130 uh. So what we did is" }, { "startMs": 231519, "text": "from 120 to $130 uh. So what we did is we came up with this cutoff which was a" }, { "startMs": 233350, "text": "we came up with this cutoff which was a" }, { "startMs": 233360, "text": "we came up with this cutoff which was a couple weeks before the announcement and" }, { "startMs": 235750, "text": "couple weeks before the announcement and" }, { "startMs": 235760, "text": "couple weeks before the announcement and we took an open source model and we gave" }, { "startMs": 238229, "text": "we took an open source model and we gave" }, { "startMs": 238239, "text": "we took an open source model and we gave it access to all of these documents from" }, { "startMs": 240470, "text": "it access to all of these documents from" }, { "startMs": 240480, "text": "it access to all of these documents from Costco uh up until that point and asked" }, { "startMs": 243509, "text": "Costco uh up until that point and asked" }, { "startMs": 243519, "text": "Costco uh up until that point and asked it like hey if Costco you know increases" }, { "startMs": 245910, "text": "it like hey if Costco you know increases" }, { "startMs": 245920, "text": "it like hey if Costco you know increases its membership fee um what are the the" }, { "startMs": 249190, "text": "its membership fee um what are the the" }, { "startMs": 249200, "text": "its membership fee um what are the the the revenue and the profit that you" }, { "startMs": 251110, "text": "the revenue and the profit that you" }, { "startMs": 251120, "text": "the revenue and the profit that you would uh forecast. So we actually" }, { "startMs": 252869, "text": "would uh forecast. So we actually" }, { "startMs": 252879, "text": "would uh forecast. So we actually started doing this with a couple of" }, { "startMs": 254630, "text": "started doing this with a couple of" }, { "startMs": 254640, "text": "started doing this with a couple of recent open source models like GLM 5.3" }, { "startMs": 256710, "text": "recent open source models like GLM 5.3" }, { "startMs": 256720, "text": "recent open source models like GLM 5.3 and Kimmy K3. But it turns out they" }, { "startMs": 258550, "text": "and Kimmy K3. But it turns out they" }, { "startMs": 258560, "text": "and Kimmy K3. But it turns out they already knew that Costco actually" }, { "startMs": 260229, "text": "already knew that Costco actually" }, { "startMs": 260239, "text": "already knew that Costco actually increased their prices um because it was" }, { "startMs": 262230, "text": "increased their prices um because it was" }, { "startMs": 262240, "text": "increased their prices um because it was already in their training data, right?" }, { "startMs": 263990, "text": "already in their training data, right?" }, { "startMs": 264000, "text": "already in their training data, right? So we had to go and find, you know," }, { "startMs": 266150, "text": "So we had to go and find, you know," }, { "startMs": 266160, "text": "So we had to go and find, you know, older open models who were trained, you" }, { "startMs": 268629, "text": "older open models who were trained, you" }, { "startMs": 268639, "text": "older open models who were trained, you know, up until before July 2024. So we" }, { "startMs": 271350, "text": "know, up until before July 2024. So we" }, { "startMs": 271360, "text": "know, up until before July 2024. So we found GBT OSS120B and Llama 3.3 uh7B. Uh" }, { "startMs": 275749, "text": "found GBT OSS120B and Llama 3.3 uh7B. Uh" }, { "startMs": 275759, "text": "found GBT OSS120B and Llama 3.3 uh7B. Uh and we this is an example of a question" }, { "startMs": 277350, "text": "and we this is an example of a question" }, { "startMs": 277360, "text": "and we this is an example of a question we asked it. Hey, did Costco change its" }, { "startMs": 279030, "text": "we asked it. Hey, did Costco change its" }, { "startMs": 279040, "text": "we asked it. Hey, did Costco change its membership fees in 2024? The newer" }, { "startMs": 281270, "text": "membership fees in 2024? The newer" }, { "startMs": 281280, "text": "membership fees in 2024? The newer models uh knew the answer. The older" }, { "startMs": 283189, "text": "models uh knew the answer. The older" }, { "startMs": 283199, "text": "models uh knew the answer. The older models didn't. And so we wanted to give" }, { "startMs": 285510, "text": "models didn't. And so we wanted to give" }, { "startMs": 285520, "text": "models didn't. And so we wanted to give we wanted to use these older models" }, { "startMs": 286870, "text": "we wanted to use these older models" }, { "startMs": 286880, "text": "we wanted to use these older models because you know you want a completely" }, { "startMs": 288070, "text": "because you know you want a completely" }, { "startMs": 288080, "text": "because you know you want a completely unbiased uh uh point of view here. So" }, { "startMs": 290230, "text": "unbiased uh uh point of view here. So" }, { "startMs": 290240, "text": "unbiased uh uh point of view here. So that's kind of the first lesson. If" }, { "startMs": 291350, "text": "that's kind of the first lesson. If" }, { "startMs": 291360, "text": "that's kind of the first lesson. If you're trying to do any sort of" }, { "startMs": 292230, "text": "you're trying to do any sort of" }, { "startMs": 292240, "text": "you're trying to do any sort of forecasting, you need to try to find a" }, { "startMs": 293830, "text": "forecasting, you need to try to find a" }, { "startMs": 293840, "text": "forecasting, you need to try to find a model whose training data ends before" }, { "startMs": 296070, "text": "model whose training data ends before" }, { "startMs": 296080, "text": "model whose training data ends before the event that you're actually trying to" }, { "startMs": 297510, "text": "the event that you're actually trying to" }, { "startMs": 297520, "text": "the event that you're actually trying to forecast. Um and uh for you know open" }, { "startMs": 300629, "text": "forecast. Um and uh for you know open" }, { "startMs": 300639, "text": "forecast. Um and uh for you know open weight models are actually really good" }, { "startMs": 301749, "text": "weight models are actually really good" }, { "startMs": 301759, "text": "weight models are actually really good for for this kind of use case. Um so" }, { "startMs": 304150, "text": "for for this kind of use case. Um so" }, { "startMs": 304160, "text": "for for this kind of use case. Um so this is kind of how everything works. We" }, { "startMs": 306070, "text": "this is kind of how everything works. We" }, { "startMs": 306080, "text": "this is kind of how everything works. We went and scraped about 214 uh public" }, { "startMs": 308870, "text": "went and scraped about 214 uh public" }, { "startMs": 308880, "text": "went and scraped about 214 uh public filings from Costco. uh you know," }, { "startMs": 311029, "text": "filings from Costco. uh you know," }, { "startMs": 311039, "text": "filings from Costco. uh you know, quarterly and yearly earning reports," }, { "startMs": 312790, "text": "quarterly and yearly earning reports," }, { "startMs": 312800, "text": "quarterly and yearly earning reports, sales reports, um all of that. We uh" }, { "startMs": 316310, "text": "sales reports, um all of that. We uh" }, { "startMs": 316320, "text": "sales reports, um all of that. We uh ended up making it into a list of about" }, { "startMs": 318469, "text": "ended up making it into a list of about" }, { "startMs": 318479, "text": "ended up making it into a list of about 3,000 facts that were all timestamped" }, { "startMs": 320629, "text": "3,000 facts that were all timestamped" }, { "startMs": 320639, "text": "3,000 facts that were all timestamped and and then we filtered these facts to" }, { "startMs": 323029, "text": "and and then we filtered these facts to" }, { "startMs": 323039, "text": "and and then we filtered these facts to only the facts available before our" }, { "startMs": 324710, "text": "only the facts available before our" }, { "startMs": 324720, "text": "only the facts available before our cutoff date. We gave it to a model and" }, { "startMs": 326950, "text": "cutoff date. We gave it to a model and" }, { "startMs": 326960, "text": "cutoff date. We gave it to a model and we actually spun up um 16 sessions of of" }, { "startMs": 329830, "text": "we actually spun up um 16 sessions of of" }, { "startMs": 329840, "text": "we actually spun up um 16 sessions of of GPTOSs 12B. We had the model actually" }, { "startMs": 333029, "text": "GPTOSs 12B. We had the model actually" }, { "startMs": 333039, "text": "GPTOSs 12B. We had the model actually write a bunch of formulas and write this" }, { "startMs": 334790, "text": "write a bunch of formulas and write this" }, { "startMs": 334800, "text": "write a bunch of formulas and write this financial model. we ran some checks and" }, { "startMs": 336870, "text": "financial model. we ran some checks and" }, { "startMs": 336880, "text": "financial model. we ran some checks and then we picked uh the from those 16 runs" }, { "startMs": 339510, "text": "then we picked uh the from those 16 runs" }, { "startMs": 339520, "text": "then we picked uh the from those 16 runs we picked the ones that actually passed" }, { "startMs": 341189, "text": "we picked the ones that actually passed" }, { "startMs": 341199, "text": "we picked the ones that actually passed uh our checks. So in in this case like" }, { "startMs": 343990, "text": "uh our checks. So in in this case like" }, { "startMs": 344000, "text": "uh our checks. So in in this case like what does the model actually do right in" }, { "startMs": 345590, "text": "what does the model actually do right in" }, { "startMs": 345600, "text": "what does the model actually do right in this previous slide where we we gave it" }, { "startMs": 347670, "text": "this previous slide where we we gave it" }, { "startMs": 347680, "text": "this previous slide where we we gave it all these facts. What does the model" }, { "startMs": 349350, "text": "all these facts. What does the model" }, { "startMs": 349360, "text": "all these facts. What does the model output here? So we actually had the" }, { "startMs": 350950, "text": "output here? So we actually had the" }, { "startMs": 350960, "text": "output here? So we actually had the model output formulas here um in and and" }, { "startMs": 354070, "text": "model output formulas here um in and and" }, { "startMs": 354080, "text": "model output formulas here um in and and so an example of this is all the way to" }, { "startMs": 356310, "text": "so an example of this is all the way to" }, { "startMs": 356320, "text": "so an example of this is all the way to the left we say like we account for" }, { "startMs": 358070, "text": "the left we say like we account for" }, { "startMs": 358080, "text": "the left we say like we account for membership fee revenue on a deferred" }, { "startMs": 359590, "text": "membership fee revenue on a deferred" }, { "startMs": 359600, "text": "membership fee revenue on a deferred basis. This is straight from Costco's" }, { "startMs": 361029, "text": "basis. This is straight from Costco's" }, { "startMs": 361039, "text": "basis. This is straight from Costco's quarterly report. We give that to the" }, { "startMs": 362469, "text": "quarterly report. We give that to the" }, { "startMs": 362479, "text": "quarterly report. We give that to the model and it constructs this uh uh" }, { "startMs": 364870, "text": "model and it constructs this uh uh" }, { "startMs": 364880, "text": "model and it constructs this uh uh structure JSON where it says you know" }, { "startMs": 366790, "text": "structure JSON where it says you know" }, { "startMs": 366800, "text": "structure JSON where it says you know the operand is ratable the term is 52" }, { "startMs": 368870, "text": "the operand is ratable the term is 52" }, { "startMs": 368880, "text": "the operand is ratable the term is 52 weeks. And so what this means is" }, { "startMs": 370230, "text": "weeks. And so what this means is" }, { "startMs": 370240, "text": "weeks. And so what this means is basically, you know, even if a user" }, { "startMs": 371990, "text": "basically, you know, even if a user" }, { "startMs": 372000, "text": "basically, you know, even if a user comes in and pays a $65 membership fee," }, { "startMs": 374469, "text": "comes in and pays a $65 membership fee," }, { "startMs": 374479, "text": "comes in and pays a $65 membership fee, it's actually spread across 52 weeks." }, { "startMs": 376870, "text": "it's actually spread across 52 weeks." }, { "startMs": 376880, "text": "it's actually spread across 52 weeks. Um, and so this is really really" }, { "startMs": 378070, "text": "Um, and so this is really really" }, { "startMs": 378080, "text": "Um, and so this is really really important when you're doing uh uh uh" }, { "startMs": 379670, "text": "important when you're doing uh uh uh" }, { "startMs": 379680, "text": "important when you're doing uh uh uh forecasting to understand how to to to" }, { "startMs": 382950, "text": "forecasting to understand how to to to" }, { "startMs": 382960, "text": "forecasting to understand how to to to forecast. And so we had the model" }, { "startMs": 384469, "text": "forecast. And so we had the model" }, { "startMs": 384479, "text": "forecast. And so we had the model construct a list of all of these" }, { "startMs": 386390, "text": "construct a list of all of these" }, { "startMs": 386400, "text": "construct a list of all of these formulas. And so the main takeaway here" }, { "startMs": 387990, "text": "formulas. And so the main takeaway here" }, { "startMs": 388000, "text": "formulas. And so the main takeaway here is like have models write structured" }, { "startMs": 389830, "text": "is like have models write structured" }, { "startMs": 389840, "text": "is like have models write structured data instead of um having them, you" }, { "startMs": 392070, "text": "data instead of um having them, you" }, { "startMs": 392080, "text": "data instead of um having them, you know, do the math and do and do" }, { "startMs": 393430, "text": "know, do the math and do and do" }, { "startMs": 393440, "text": "know, do the math and do and do everything all at once. Um the model" }, { "startMs": 395590, "text": "everything all at once. Um the model" }, { "startMs": 395600, "text": "everything all at once. Um the model also came up with a bunch of" }, { "startMs": 396710, "text": "also came up with a bunch of" }, { "startMs": 396720, "text": "also came up with a bunch of assumptions. Uh and so it had to guess" }, { "startMs": 398230, "text": "assumptions. Uh and so it had to guess" }, { "startMs": 398240, "text": "assumptions. Uh and so it had to guess on a few things. So, for example, what" }, { "startMs": 399990, "text": "on a few things. So, for example, what" }, { "startMs": 400000, "text": "on a few things. So, for example, what percentage of people are in the standard" }, { "startMs": 401749, "text": "percentage of people are in the standard" }, { "startMs": 401759, "text": "percentage of people are in the standard tier versus the executive tier? Um, how" }, { "startMs": 404710, "text": "tier versus the executive tier? Um, how" }, { "startMs": 404720, "text": "tier versus the executive tier? Um, how many members were in Canada can US and" }, { "startMs": 406309, "text": "many members were in Canada can US and" }, { "startMs": 406319, "text": "many members were in Canada can US and Canada and impacted by this change. U," }, { "startMs": 408230, "text": "Canada and impacted by this change. U," }, { "startMs": 408240, "text": "Canada and impacted by this change. U, so it made a few assumptions. My dad's" }, { "startMs": 410070, "text": "so it made a few assumptions. My dad's" }, { "startMs": 410080, "text": "so it made a few assumptions. My dad's going to go over some of these" }, { "startMs": 410870, "text": "going to go over some of these" }, { "startMs": 410880, "text": "going to go over some of these assumptions later and why some of them" }, { "startMs": 412309, "text": "assumptions later and why some of them" }, { "startMs": 412319, "text": "assumptions later and why some of them may or may not be wrong. Um, and and" }, { "startMs": 414390, "text": "may or may not be wrong. Um, and and" }, { "startMs": 414400, "text": "may or may not be wrong. Um, and and here's kind of the full picture over" }, { "startMs": 415749, "text": "here's kind of the full picture over" }, { "startMs": 415759, "text": "here's kind of the full picture over here for the full financial model. On" }, { "startMs": 417270, "text": "here for the full financial model. On" }, { "startMs": 417280, "text": "here for the full financial model. On the left, you have the assumptions," }, { "startMs": 418629, "text": "the left, you have the assumptions," }, { "startMs": 418639, "text": "the left, you have the assumptions, which we went over some on the last" }, { "startMs": 419990, "text": "which we went over some on the last" }, { "startMs": 420000, "text": "which we went over some on the last slide, um, that the model came up with." }, { "startMs": 422870, "text": "slide, um, that the model came up with." }, { "startMs": 422880, "text": "slide, um, that the model came up with. And then we have 14 formulas that the" }, { "startMs": 424550, "text": "And then we have 14 formulas that the" }, { "startMs": 424560, "text": "And then we have 14 formulas that the model also came up with. Um, and and" }, { "startMs": 426469, "text": "model also came up with. Um, and and" }, { "startMs": 426479, "text": "model also came up with. Um, and and then we have a few numbers that are kind" }, { "startMs": 427909, "text": "then we have a few numbers that are kind" }, { "startMs": 427919, "text": "then we have a few numbers that are kind of directly from uh Costco's filings." }, { "startMs": 429909, "text": "of directly from uh Costco's filings." }, { "startMs": 429919, "text": "of directly from uh Costco's filings. And this is actually an Excel" }, { "startMs": 431029, "text": "And this is actually an Excel" }, { "startMs": 431039, "text": "And this is actually an Excel spreadsheet that that I created. I'm" }, { "startMs": 432550, "text": "spreadsheet that that I created. I'm" }, { "startMs": 432560, "text": "spreadsheet that that I created. I'm kind of just trying to visualize it in a" }, { "startMs": 433990, "text": "kind of just trying to visualize it in a" }, { "startMs": 434000, "text": "kind of just trying to visualize it in a way uh where where it's uh it's a little" }, { "startMs": 436390, "text": "way uh where where it's uh it's a little" }, { "startMs": 436400, "text": "way uh where where it's uh it's a little bit nicer to to look at. Um, and so" }, { "startMs": 438790, "text": "bit nicer to to look at. Um, and so" }, { "startMs": 438800, "text": "bit nicer to to look at. Um, and so after we had the model create this" }, { "startMs": 440390, "text": "after we had the model create this" }, { "startMs": 440400, "text": "after we had the model create this financial model, we uh actually had it" }, { "startMs": 444309, "text": "financial model, we uh actually had it" }, { "startMs": 444319, "text": "financial model, we uh actually had it go through a bunch of these checks. Uh," }, { "startMs": 446230, "text": "go through a bunch of these checks. Uh," }, { "startMs": 446240, "text": "go through a bunch of these checks. Uh, and so there's just basic checks. For" }, { "startMs": 447589, "text": "and so there's just basic checks. For" }, { "startMs": 447599, "text": "and so there's just basic checks. For example, a formula can't depend on" }, { "startMs": 449029, "text": "example, a formula can't depend on" }, { "startMs": 449039, "text": "example, a formula can't depend on itself. For example, raising the" }, { "startMs": 450390, "text": "itself. For example, raising the" }, { "startMs": 450400, "text": "itself. For example, raising the membership fee needs to raise revenue." }, { "startMs": 452390, "text": "membership fee needs to raise revenue." }, { "startMs": 452400, "text": "membership fee needs to raise revenue. you know, just making sure that like" }, { "startMs": 453749, "text": "you know, just making sure that like" }, { "startMs": 453759, "text": "you know, just making sure that like some a few basic things were were" }, { "startMs": 455510, "text": "some a few basic things were were" }, { "startMs": 455520, "text": "some a few basic things were were correct. And whenever a model whenever" }, { "startMs": 457990, "text": "correct. And whenever a model whenever" }, { "startMs": 458000, "text": "correct. And whenever a model whenever one of those runs, because remember we" }, { "startMs": 459430, "text": "one of those runs, because remember we" }, { "startMs": 459440, "text": "one of those runs, because remember we had these 16 runs, whenever one of those" }, { "startMs": 461189, "text": "had these 16 runs, whenever one of those" }, { "startMs": 461199, "text": "had these 16 runs, whenever one of those runs got one of these things wrong, we" }, { "startMs": 462790, "text": "runs got one of these things wrong, we" }, { "startMs": 462800, "text": "runs got one of these things wrong, we ended up passing the error back to the" }, { "startMs": 464710, "text": "ended up passing the error back to the" }, { "startMs": 464720, "text": "ended up passing the error back to the model uh up to two different times uh so" }, { "startMs": 467510, "text": "model uh up to two different times uh so" }, { "startMs": 467520, "text": "model uh up to two different times uh so that it kind of tried to self-correct" }, { "startMs": 469749, "text": "that it kind of tried to self-correct" }, { "startMs": 469759, "text": "that it kind of tried to self-correct itself. And what was interesting here," }, { "startMs": 471430, "text": "itself. And what was interesting here," }, { "startMs": 471440, "text": "itself. And what was interesting here, so these are actually the the 16" }, { "startMs": 473110, "text": "so these are actually the the 16" }, { "startMs": 473120, "text": "so these are actually the the 16 different runs that we have. They're all" }, { "startMs": 474629, "text": "different runs that we have. They're all" }, { "startMs": 474639, "text": "different runs that we have. They're all running. They're all creating their" }, { "startMs": 475670, "text": "running. They're all creating their" }, { "startMs": 475680, "text": "running. They're all creating their financial model. They're writing the the" }, { "startMs": 477510, "text": "financial model. They're writing the the" }, { "startMs": 477520, "text": "financial model. They're writing the the formulas. We're passing the errors back." }, { "startMs": 479830, "text": "formulas. We're passing the errors back." }, { "startMs": 479840, "text": "formulas. We're passing the errors back. And what ended up happening is something" }, { "startMs": 481189, "text": "And what ended up happening is something" }, { "startMs": 481199, "text": "And what ended up happening is something interesting where we had a bunch of them" }, { "startMs": 482790, "text": "interesting where we had a bunch of them" }, { "startMs": 482800, "text": "interesting where we had a bunch of them fail anyway. Um because they just failed" }, { "startMs": 485110, "text": "fail anyway. Um because they just failed" }, { "startMs": 485120, "text": "fail anyway. Um because they just failed more than the two times we we we gave" }, { "startMs": 487029, "text": "more than the two times we we we gave" }, { "startMs": 487039, "text": "more than the two times we we we gave them back their their error. Um and so" }, { "startMs": 489510, "text": "them back their their error. Um and so" }, { "startMs": 489520, "text": "them back their their error. Um and so we ended up uh using these five that" }, { "startMs": 491589, "text": "we ended up uh using these five that" }, { "startMs": 491599, "text": "we ended up uh using these five that ended up passing uh for for our uh" }, { "startMs": 493909, "text": "ended up passing uh for for our uh" }, { "startMs": 493919, "text": "ended up passing uh for for our uh financial forecast in general. And" }, { "startMs": 495350, "text": "financial forecast in general. And" }, { "startMs": 495360, "text": "financial forecast in general. And that's kind of another lesson. We ended" }, { "startMs": 497110, "text": "that's kind of another lesson. We ended" }, { "startMs": 497120, "text": "that's kind of another lesson. We ended up starting with just like one prompt" }, { "startMs": 498550, "text": "up starting with just like one prompt" }, { "startMs": 498560, "text": "up starting with just like one prompt that we kept tuning and tuning and" }, { "startMs": 499830, "text": "that we kept tuning and tuning and" }, { "startMs": 499840, "text": "that we kept tuning and tuning and tuning. We're like, \"Oh, like surely" }, { "startMs": 501430, "text": "tuning. We're like, \"Oh, like surely" }, { "startMs": 501440, "text": "tuning. We're like, \"Oh, like surely like we're going to give this a GPT" }, { "startMs": 502790, "text": "like we're going to give this a GPT" }, { "startMs": 502800, "text": "like we're going to give this a GPT OSS120B and it's going to pass, you" }, { "startMs": 504550, "text": "OSS120B and it's going to pass, you" }, { "startMs": 504560, "text": "OSS120B and it's going to pass, you know, all the checks.\" Um, but that" }, { "startMs": 506150, "text": "know, all the checks.\" Um, but that" }, { "startMs": 506160, "text": "know, all the checks.\" Um, but that ended up like just taking so much time" }, { "startMs": 507749, "text": "ended up like just taking so much time" }, { "startMs": 507759, "text": "ended up like just taking so much time that we were like, what if we just spun" }, { "startMs": 509189, "text": "that we were like, what if we just spun" }, { "startMs": 509199, "text": "that we were like, what if we just spun up, you know, 20 16 different runs. Uh," }, { "startMs": 512149, "text": "up, you know, 20 16 different runs. Uh," }, { "startMs": 512159, "text": "up, you know, 20 16 different runs. Uh, and in this case, it works really well" }, { "startMs": 513350, "text": "and in this case, it works really well" }, { "startMs": 513360, "text": "and in this case, it works really well because the model is really cheap. So," }, { "startMs": 514709, "text": "because the model is really cheap. So," }, { "startMs": 514719, "text": "because the model is really cheap. So, it only cost us 19 cents to spin up" }, { "startMs": 516550, "text": "it only cost us 19 cents to spin up" }, { "startMs": 516560, "text": "it only cost us 19 cents to spin up these like 16 different runs and we had" }, { "startMs": 518469, "text": "these like 16 different runs and we had" }, { "startMs": 518479, "text": "these like 16 different runs and we had more of them that that passed this way." }, { "startMs": 521589, "text": "more of them that that passed this way." }, { "startMs": 521599, "text": "more of them that that passed this way. So, this is the million-dollar question" }, { "startMs": 523670, "text": "So, this is the million-dollar question" }, { "startMs": 523680, "text": "So, this is the million-dollar question that we asked, right? If cost what if" }, { "startMs": 525750, "text": "that we asked, right? If cost what if" }, { "startMs": 525760, "text": "that we asked, right? If cost what if Costco raises its fee uh this September," }, { "startMs": 528310, "text": "Costco raises its fee uh this September," }, { "startMs": 528320, "text": "Costco raises its fee uh this September, what happens to its actual revenue? So" }, { "startMs": 530949, "text": "what happens to its actual revenue? So" }, { "startMs": 530959, "text": "what happens to its actual revenue? So we had GLM 5.3 translated into just like" }, { "startMs": 533430, "text": "we had GLM 5.3 translated into just like" }, { "startMs": 533440, "text": "we had GLM 5.3 translated into just like some structured output that we can feed" }, { "startMs": 534870, "text": "some structured output that we can feed" }, { "startMs": 534880, "text": "some structured output that we can feed into the financial model. And then the" }, { "startMs": 536550, "text": "into the financial model. And then the" }, { "startMs": 536560, "text": "into the financial model. And then the model gave us this number of like it" }, { "startMs": 538710, "text": "model gave us this number of like it" }, { "startMs": 538720, "text": "model gave us this number of like it predicted that uh profit would increase" }, { "startMs": 541750, "text": "predicted that uh profit would increase" }, { "startMs": 541760, "text": "predicted that uh profit would increase by $560 million over two years. Uh and" }, { "startMs": 545110, "text": "by $560 million over two years. Uh and" }, { "startMs": 545120, "text": "by $560 million over two years. Uh and so this is it just in the app. We send" }, { "startMs": 547430, "text": "so this is it just in the app. We send" }, { "startMs": 547440, "text": "so this is it just in the app. We send this over uh GLM 5.3 interprets it. It" }, { "startMs": 550070, "text": "this over uh GLM 5.3 interprets it. It" }, { "startMs": 550080, "text": "this over uh GLM 5.3 interprets it. It sends it as structured data to the" }, { "startMs": 551750, "text": "sends it as structured data to the" }, { "startMs": 551760, "text": "sends it as structured data to the model. And remember the model is just a" }, { "startMs": 553509, "text": "model. And remember the model is just a" }, { "startMs": 553519, "text": "model. And remember the model is just a financial model. It takes 10" }, { "startMs": 554630, "text": "financial model. It takes 10" }, { "startMs": 554640, "text": "financial model. It takes 10 milliseconds to run uh because the mo" }, { "startMs": 556470, "text": "milliseconds to run uh because the mo" }, { "startMs": 556480, "text": "milliseconds to run uh because the mo because the uh our GPOSS model already" }, { "startMs": 558790, "text": "because the uh our GPOSS model already" }, { "startMs": 558800, "text": "because the uh our GPOSS model already made it. And here's that 560 million uh" }, { "startMs": 561110, "text": "made it. And here's that 560 million uh" }, { "startMs": 561120, "text": "made it. And here's that 560 million uh number right there. So, um this is it" }, { "startMs": 563910, "text": "number right there. So, um this is it" }, { "startMs": 563920, "text": "number right there. So, um this is it just like visualized on a scale. The" }, { "startMs": 565750, "text": "just like visualized on a scale. The" }, { "startMs": 565760, "text": "just like visualized on a scale. The blue line is the predicted uh uh uh" }, { "startMs": 568470, "text": "blue line is the predicted uh uh uh" }, { "startMs": 568480, "text": "blue line is the predicted uh uh uh profit and orange is actually the real" }, { "startMs": 571269, "text": "profit and orange is actually the real" }, { "startMs": 571279, "text": "profit and orange is actually the real profit extracted from Costco's uh" }, { "startMs": 573110, "text": "profit extracted from Costco's uh" }, { "startMs": 573120, "text": "profit extracted from Costco's uh earning reports. And so you can see we" }, { "startMs": 574790, "text": "earning reports. And so you can see we" }, { "startMs": 574800, "text": "earning reports. And so you can see we were accurate to about 4.4% uh which is" }, { "startMs": 578070, "text": "were accurate to about 4.4% uh which is" }, { "startMs": 578080, "text": "were accurate to about 4.4% uh which is the average of the five builds. The" }, { "startMs": 579350, "text": "the average of the five builds. The" }, { "startMs": 579360, "text": "the average of the five builds. The builds gave us between 0.7 and 5.2%. So" }, { "startMs": 583030, "text": "builds gave us between 0.7 and 5.2%. So" }, { "startMs": 583040, "text": "builds gave us between 0.7 and 5.2%. So again, I thought this was pretty good," }, { "startMs": 584310, "text": "again, I thought this was pretty good," }, { "startMs": 584320, "text": "again, I thought this was pretty good, right? We had the model build uh we had" }, { "startMs": 586550, "text": "right? We had the model build uh we had" }, { "startMs": 586560, "text": "right? We had the model build uh we had our AI model build this financial model." }, { "startMs": 588870, "text": "our AI model build this financial model." }, { "startMs": 588880, "text": "our AI model build this financial model. We ran it. It was accurate within 4.4%." }, { "startMs": 591590, "text": "We ran it. It was accurate within 4.4%." }, { "startMs": 591600, "text": "We ran it. It was accurate within 4.4%. Uh and then I showed it to my dad. Yeah," }, { "startMs": 594150, "text": "Uh and then I showed it to my dad. Yeah," }, { "startMs": 594160, "text": "Uh and then I showed it to my dad. Yeah, definitely. It's amazing to see that AI" }, { "startMs": 595990, "text": "definitely. It's amazing to see that AI" }, { "startMs": 596000, "text": "definitely. It's amazing to see that AI can can get to financial models which is" }, { "startMs": 598150, "text": "can can get to financial models which is" }, { "startMs": 598160, "text": "can can get to financial models which is the you know the area of accountants and" }, { "startMs": 599910, "text": "the you know the area of accountants and" }, { "startMs": 599920, "text": "the you know the area of accountants and and financial people but with with very" }, { "startMs": 602550, "text": "and financial people but with with very" }, { "startMs": 602560, "text": "and financial people but with with very small guidance but the first thing you" }, { "startMs": 604550, "text": "small guidance but the first thing you" }, { "startMs": 604560, "text": "small guidance but the first thing you look at when you try to look at a" }, { "startMs": 606230, "text": "look at when you try to look at a" }, { "startMs": 606240, "text": "look at when you try to look at a financial model is to audit it is to" }, { "startMs": 608230, "text": "financial model is to audit it is to" }, { "startMs": 608240, "text": "financial model is to audit it is to look at assumptions. So these are some" }, { "startMs": 610310, "text": "look at assumptions. So these are some" }, { "startMs": 610320, "text": "look at assumptions. So these are some of the assumptions that were used in" }, { "startMs": 611750, "text": "of the assumptions that were used in" }, { "startMs": 611760, "text": "of the assumptions that were used in Hassan's model and uh I just want to" }, { "startMs": 614389, "text": "Hassan's model and uh I just want to" }, { "startMs": 614399, "text": "Hassan's model and uh I just want to talk your attention to two things. The" }, { "startMs": 615750, "text": "talk your attention to two things. The" }, { "startMs": 615760, "text": "talk your attention to two things. The executive share like Hassan mentioned" }, { "startMs": 617430, "text": "executive share like Hassan mentioned" }, { "startMs": 617440, "text": "executive share like Hassan mentioned before. Uh it was um you know big swings" }, { "startMs": 620310, "text": "before. Uh it was um you know big swings" }, { "startMs": 620320, "text": "before. Uh it was um you know big swings there. Um US Canada the last um you know" }, { "startMs": 623670, "text": "there. Um US Canada the last um you know" }, { "startMs": 623680, "text": "there. Um US Canada the last um you know um assumption did not move um sales and" }, { "startMs": 627910, "text": "um assumption did not move um sales and" }, { "startMs": 627920, "text": "um assumption did not move um sales and um and fees with much with zero. So" }, { "startMs": 631030, "text": "um and fees with much with zero. So" }, { "startMs": 631040, "text": "um and fees with much with zero. So basically that's a lever that is not" }, { "startMs": 632870, "text": "basically that's a lever that is not" }, { "startMs": 632880, "text": "basically that's a lever that is not working and it should be and something" }, { "startMs": 634630, "text": "working and it should be and something" }, { "startMs": 634640, "text": "working and it should be and something was wrong. Something went wrong there" }, { "startMs": 636550, "text": "was wrong. Something went wrong there" }, { "startMs": 636560, "text": "was wrong. Something went wrong there and it should be fixed." }, { "startMs": 639430, "text": "and it should be fixed." }, { "startMs": 639440, "text": "and it should be fixed. Um so these are why the executive share" }, { "startMs": 642870, "text": "Um so these are why the executive share" }, { "startMs": 642880, "text": "Um so these are why the executive share for example the executive share the 15%" }, { "startMs": 644949, "text": "for example the executive share the 15%" }, { "startMs": 644959, "text": "for example the executive share the 15% the mother took 15%. Um and the problem" }, { "startMs": 647829, "text": "the mother took 15%. Um and the problem" }, { "startMs": 647839, "text": "the mother took 15%. Um and the problem is that the mother had just one citation" }, { "startMs": 650230, "text": "is that the mother had just one citation" }, { "startMs": 650240, "text": "is that the mother had just one citation increased penetration of executive" }, { "startMs": 651910, "text": "increased penetration of executive" }, { "startMs": 651920, "text": "increased penetration of executive membership but there was no number to" }, { "startMs": 654150, "text": "membership but there was no number to" }, { "startMs": 654160, "text": "membership but there was no number to back it up. So that's what the auditors" }, { "startMs": 656470, "text": "back it up. So that's what the auditors" }, { "startMs": 656480, "text": "back it up. So that's what the auditors looking at at how are all the facts" }, { "startMs": 658790, "text": "looking at at how are all the facts" }, { "startMs": 658800, "text": "looking at at how are all the facts backed up with you know with with real" }, { "startMs": 660790, "text": "backed up with you know with with real" }, { "startMs": 660800, "text": "backed up with you know with with real numbers. So the citation is real but the" }, { "startMs": 663190, "text": "numbers. So the citation is real but the" }, { "startMs": 663200, "text": "numbers. So the citation is real but the problem and the the the number was ma" }, { "startMs": 665190, "text": "problem and the the the number was ma" }, { "startMs": 665200, "text": "problem and the the the number was ma was made up by AI and not only that AI" }, { "startMs": 668470, "text": "was made up by AI and not only that AI" }, { "startMs": 668480, "text": "was made up by AI and not only that AI also draw the boundaries of that" }, { "startMs": 670389, "text": "also draw the boundaries of that" }, { "startMs": 670399, "text": "also draw the boundaries of that assumption which is uh zero to um 0 to 5" }, { "startMs": 674389, "text": "assumption which is uh zero to um 0 to 5" }, { "startMs": 674399, "text": "assumption which is uh zero to um 0 to 5 to 30%. But we will see that it's not" }, { "startMs": 677190, "text": "to 30%. But we will see that it's not" }, { "startMs": 677200, "text": "to 30%. But we will see that it's not correct actually um the that's what the" }, { "startMs": 680630, "text": "correct actually um the that's what the" }, { "startMs": 680640, "text": "correct actually um the that's what the AI did 5 to 30% but the problem is the" }, { "startMs": 684710, "text": "AI did 5 to 30% but the problem is the" }, { "startMs": 684720, "text": "AI did 5 to 30% but the problem is the real number that was already disclosed" }, { "startMs": 686710, "text": "real number that was already disclosed" }, { "startMs": 686720, "text": "real number that was already disclosed in the in in the filings um was 45%. So" }, { "startMs": 690870, "text": "in the in in the filings um was 45%. So" }, { "startMs": 690880, "text": "in the in in the filings um was 45%. So in this case AI failed to um you know um" }, { "startMs": 694949, "text": "in this case AI failed to um you know um" }, { "startMs": 694959, "text": "in this case AI failed to um you know um identify that number from the from the" }, { "startMs": 696710, "text": "identify that number from the from the" }, { "startMs": 696720, "text": "identify that number from the from the filing. So that's one uh one problem and" }, { "startMs": 699670, "text": "filing. So that's one uh one problem and" }, { "startMs": 699680, "text": "filing. So that's one uh one problem and um the surprise why did it land correct" }, { "startMs": 702710, "text": "um the surprise why did it land correct" }, { "startMs": 702720, "text": "um the surprise why did it land correct like hasn't mentioned it's 4.4% 4% which" }, { "startMs": 705430, "text": "like hasn't mentioned it's 4.4% 4% which" }, { "startMs": 705440, "text": "like hasn't mentioned it's 4.4% 4% which is a okay uh performance because of two" }, { "startMs": 710389, "text": "is a okay uh performance because of two" }, { "startMs": 710399, "text": "is a okay uh performance because of two errors that canceled out two assumptions" }, { "startMs": 712310, "text": "errors that canceled out two assumptions" }, { "startMs": 712320, "text": "errors that canceled out two assumptions that canceled out and the first one was" }, { "startMs": 714790, "text": "that canceled out and the first one was" }, { "startMs": 714800, "text": "that canceled out and the first one was an overstatement of the base how many" }, { "startMs": 717030, "text": "an overstatement of the base how many" }, { "startMs": 717040, "text": "an overstatement of the base how many people instead of applying the um" }, { "startMs": 719030, "text": "people instead of applying the um" }, { "startMs": 719040, "text": "people instead of applying the um increase on just the US Canada portion" }, { "startMs": 722550, "text": "increase on just the US Canada portion" }, { "startMs": 722560, "text": "increase on just the US Canada portion of the members the model applied it to" }, { "startMs": 725030, "text": "of the members the model applied it to" }, { "startMs": 725040, "text": "of the members the model applied it to international to to the whole population" }, { "startMs": 727030, "text": "international to to the whole population" }, { "startMs": 727040, "text": "international to to the whole population actually and the second thing that was" }, { "startMs": 729190, "text": "actually and the second thing that was" }, { "startMs": 729200, "text": "actually and the second thing that was understated it was too low it's the" }, { "startMs": 731829, "text": "understated it was too low it's the" }, { "startMs": 731839, "text": "understated it was too low it's the percentage uh instead of 45% which is" }, { "startMs": 734470, "text": "percentage uh instead of 45% which is" }, { "startMs": 734480, "text": "percentage uh instead of 45% which is the real number. Um AI made up this 15%" }, { "startMs": 737910, "text": "the real number. Um AI made up this 15%" }, { "startMs": 737920, "text": "the real number. Um AI made up this 15% and that was too low. It landed $69" }, { "startMs": 741110, "text": "and that was too low. It landed $69" }, { "startMs": 741120, "text": "and that was too low. It landed $69 which is close to 66 but um again the" }, { "startMs": 743990, "text": "which is close to 66 but um again the" }, { "startMs": 744000, "text": "which is close to 66 but um again the model behind it was wrong. And this is" }, { "startMs": 747350, "text": "model behind it was wrong. And this is" }, { "startMs": 747360, "text": "model behind it was wrong. And this is um this is what Hassan was mentioning" }, { "startMs": 749269, "text": "um this is what Hassan was mentioning" }, { "startMs": 749279, "text": "um this is what Hassan was mentioning before um that um like that led to um" }, { "startMs": 754389, "text": "before um that um like that led to um" }, { "startMs": 754399, "text": "before um that um like that led to um like the 16 the 16 models they failed" }, { "startMs": 757990, "text": "like the 16 the 16 models they failed" }, { "startMs": 758000, "text": "like the 16 the 16 models they failed and they failed um all with the same" }, { "startMs": 760389, "text": "and they failed um all with the same" }, { "startMs": 760399, "text": "and they failed um all with the same thing with big ranges from half billion" }, { "startMs": 762710, "text": "thing with big ranges from half billion" }, { "startMs": 762720, "text": "thing with big ranges from half billion to $6 billion in in terms of fee" }, { "startMs": 765750, "text": "to $6 billion in in terms of fee" }, { "startMs": 765760, "text": "to $6 billion in in terms of fee increase while the actual number that we" }, { "startMs": 768069, "text": "increase while the actual number that we" }, { "startMs": 768079, "text": "increase while the actual number that we recalculated was just $450 million. So" }, { "startMs": 771430, "text": "recalculated was just $450 million. So" }, { "startMs": 771440, "text": "recalculated was just $450 million. So that that was one of the other issues" }, { "startMs": 773670, "text": "that that was one of the other issues" }, { "startMs": 773680, "text": "that that was one of the other issues here. And then um just a trick here the" }, { "startMs": 776790, "text": "here. And then um just a trick here the" }, { "startMs": 776800, "text": "here. And then um just a trick here the way to know whether a model or um to to" }, { "startMs": 780310, "text": "way to know whether a model or um to to" }, { "startMs": 780320, "text": "way to know whether a model or um to to something is wrong if you have two" }, { "startMs": 782230, "text": "something is wrong if you have two" }, { "startMs": 782240, "text": "something is wrong if you have two things cancelling out like an auditing" }, { "startMs": 784150, "text": "things cancelling out like an auditing" }, { "startMs": 784160, "text": "things cancelling out like an auditing trick you just fix one of the two and" }, { "startMs": 786069, "text": "trick you just fix one of the two and" }, { "startMs": 786079, "text": "trick you just fix one of the two and the model will look worse. So before the" }, { "startMs": 789430, "text": "the model will look worse. So before the" }, { "startMs": 789440, "text": "the model will look worse. So before the performance was 1.7 because things were" }, { "startMs": 791829, "text": "performance was 1.7 because things were" }, { "startMs": 791839, "text": "performance was 1.7 because things were washing out canceling out and then once" }, { "startMs": 794550, "text": "washing out canceling out and then once" }, { "startMs": 794560, "text": "washing out canceling out and then once we correct just one we just we put like" }, { "startMs": 796870, "text": "we correct just one we just we put like" }, { "startMs": 796880, "text": "we correct just one we just we put like the 45% instead of 15% that was made up" }, { "startMs": 800310, "text": "the 45% instead of 15% that was made up" }, { "startMs": 800320, "text": "the 45% instead of 15% that was made up by AI suddenly the problem the error was" }, { "startMs": 803910, "text": "by AI suddenly the problem the error was" }, { "startMs": 803920, "text": "by AI suddenly the problem the error was 28%. So that's what we say luck is not a" }, { "startMs": 807509, "text": "28%. So that's what we say luck is not a" }, { "startMs": 807519, "text": "28%. So that's what we say luck is not a control in the auditing world. Usually" }, { "startMs": 810069, "text": "control in the auditing world. Usually" }, { "startMs": 810079, "text": "control in the auditing world. Usually in auditing we use three lines of" }, { "startMs": 811829, "text": "in auditing we use three lines of" }, { "startMs": 811839, "text": "in auditing we use three lines of defense. We have management controls" }, { "startMs": 813750, "text": "defense. We have management controls" }, { "startMs": 813760, "text": "defense. We have management controls which is the engineering checks that" }, { "startMs": 815269, "text": "which is the engineering checks that" }, { "startMs": 815279, "text": "which is the engineering checks that hasn't um coded and then we have the uh" }, { "startMs": 818710, "text": "hasn't um coded and then we have the uh" }, { "startMs": 818720, "text": "hasn't um coded and then we have the uh internal audit or some some internal um" }, { "startMs": 821269, "text": "internal audit or some some internal um" }, { "startMs": 821279, "text": "internal audit or some some internal um accounting checks that should be" }, { "startMs": 822550, "text": "accounting checks that should be" }, { "startMs": 822560, "text": "accounting checks that should be performed by internal um uh audit" }, { "startMs": 824550, "text": "performed by internal um uh audit" }, { "startMs": 824560, "text": "performed by internal um uh audit function and then we have a third layer" }, { "startMs": 827110, "text": "function and then we have a third layer" }, { "startMs": 827120, "text": "function and then we have a third layer which is the independent check that's" }, { "startMs": 829190, "text": "which is the independent check that's" }, { "startMs": 829200, "text": "which is the independent check that's external auditors. So these layers they" }, { "startMs": 831990, "text": "external auditors. So these layers they" }, { "startMs": 832000, "text": "external auditors. So these layers they will benefit like the model will benefit" }, { "startMs": 833750, "text": "will benefit like the model will benefit" }, { "startMs": 833760, "text": "will benefit like the model will benefit from adding layer two and three and" }, { "startMs": 835670, "text": "from adding layer two and three and" }, { "startMs": 835680, "text": "from adding layer two and three and that's the role I was playing like this" }, { "startMs": 838069, "text": "that's the role I was playing like this" }, { "startMs": 838079, "text": "that's the role I was playing like this CPA in the loop or what I call uh" }, { "startMs": 840949, "text": "CPA in the loop or what I call uh" }, { "startMs": 840959, "text": "CPA in the loop or what I call uh actually the judgment professional" }, { "startMs": 843189, "text": "actually the judgment professional" }, { "startMs": 843199, "text": "actually the judgment professional judgment in the loop um to because we" }, { "startMs": 845750, "text": "judgment in the loop um to because we" }, { "startMs": 845760, "text": "judgment in the loop um to because we can we can hardcode most of those checks" }, { "startMs": 848230, "text": "can we can hardcode most of those checks" }, { "startMs": 848240, "text": "can we can hardcode most of those checks but still we need uh we need a human um" }, { "startMs": 851189, "text": "but still we need uh we need a human um" }, { "startMs": 851199, "text": "but still we need uh we need a human um looking at that." }, { "startMs": 852150, "text": "looking at that." }, { "startMs": 852160, "text": "looking at that. Great. So, you know, as my dad said, we" }, { "startMs": 854870, "text": "Great. So, you know, as my dad said, we" }, { "startMs": 854880, "text": "Great. So, you know, as my dad said, we we had the model here. We passed it" }, { "startMs": 856550, "text": "we had the model here. We passed it" }, { "startMs": 856560, "text": "we had the model here. We passed it through engineering checks. It did" }, { "startMs": 858870, "text": "through engineering checks. It did" }, { "startMs": 858880, "text": "through engineering checks. It did pretty well. Uh, you know, we got a" }, { "startMs": 860389, "text": "pretty well. Uh, you know, we got a" }, { "startMs": 860399, "text": "pretty well. Uh, you know, we got a little bit lucky with the result, but" }, { "startMs": 861990, "text": "little bit lucky with the result, but" }, { "startMs": 862000, "text": "little bit lucky with the result, but was really missing is these like" }, { "startMs": 863110, "text": "was really missing is these like" }, { "startMs": 863120, "text": "was really missing is these like accounting checks or this like kind of" }, { "startMs": 864470, "text": "accounting checks or this like kind of" }, { "startMs": 864480, "text": "accounting checks or this like kind of human in the loop uh uh activity. So, uh" }, { "startMs": 867509, "text": "human in the loop uh uh activity. So, uh" }, { "startMs": 867519, "text": "human in the loop uh uh activity. So, uh and a lot of these accounting checks" }, { "startMs": 868550, "text": "and a lot of these accounting checks" }, { "startMs": 868560, "text": "and a lot of these accounting checks that my dad came up with, you know," }, { "startMs": 869910, "text": "that my dad came up with, you know," }, { "startMs": 869920, "text": "that my dad came up with, you know, we're were able to to codify. So, one" }, { "startMs": 872150, "text": "we're were able to to codify. So, one" }, { "startMs": 872160, "text": "we're were able to to codify. So, one was the the executive 45% like my dad" }, { "startMs": 874550, "text": "was the the executive 45% like my dad" }, { "startMs": 874560, "text": "was the the executive 45% like my dad said. Um so, it's really important to" }, { "startMs": 876470, "text": "said. Um so, it's really important to" }, { "startMs": 876480, "text": "said. Um so, it's really important to have these like domain specific checks" }, { "startMs": 878230, "text": "have these like domain specific checks" }, { "startMs": 878240, "text": "have these like domain specific checks when you're building uh these these AI" }, { "startMs": 880389, "text": "when you're building uh these these AI" }, { "startMs": 880399, "text": "when you're building uh these these AI agents. You know, I don't come from the" }, { "startMs": 881509, "text": "agents. You know, I don't come from the" }, { "startMs": 881519, "text": "agents. You know, I don't come from the from the accounting world. when I built" }, { "startMs": 882790, "text": "from the accounting world. when I built" }, { "startMs": 882800, "text": "from the accounting world. when I built this and I saw 4% I was like oh amazing." }, { "startMs": 885269, "text": "this and I saw 4% I was like oh amazing." }, { "startMs": 885279, "text": "this and I saw 4% I was like oh amazing. Um but but really it's um it's yeah this" }, { "startMs": 888949, "text": "Um but but really it's um it's yeah this" }, { "startMs": 888959, "text": "Um but but really it's um it's yeah this is really the number one thing to" }, { "startMs": 889990, "text": "is really the number one thing to" }, { "startMs": 890000, "text": "is really the number one thing to remember from this talk. If you take one" }, { "startMs": 891110, "text": "remember from this talk. If you take one" }, { "startMs": 891120, "text": "remember from this talk. If you take one thing away um having a human in the loop" }, { "startMs": 893430, "text": "thing away um having a human in the loop" }, { "startMs": 893440, "text": "thing away um having a human in the loop or ideally a subject matter expert uh in" }, { "startMs": 895829, "text": "or ideally a subject matter expert uh in" }, { "startMs": 895839, "text": "or ideally a subject matter expert uh in the loop is really really really" }, { "startMs": 897590, "text": "the loop is really really really" }, { "startMs": 897600, "text": "the loop is really really really important. Um they can take a look at" }, { "startMs": 899590, "text": "important. Um they can take a look at" }, { "startMs": 899600, "text": "important. Um they can take a look at your agent or your model or or whatever" }, { "startMs": 901430, "text": "your agent or your model or or whatever" }, { "startMs": 901440, "text": "your agent or your model or or whatever you've actually built. They can check" }, { "startMs": 902870, "text": "you've actually built. They can check" }, { "startMs": 902880, "text": "you've actually built. They can check the assumptions, check the inputs, check" }, { "startMs": 904389, "text": "the assumptions, check the inputs, check" }, { "startMs": 904399, "text": "the assumptions, check the inputs, check the outputs, make the right corrections." }, { "startMs": 906470, "text": "the outputs, make the right corrections." }, { "startMs": 906480, "text": "the outputs, make the right corrections. you can codify their checks in the model" }, { "startMs": 908629, "text": "you can codify their checks in the model" }, { "startMs": 908639, "text": "you can codify their checks in the model and it it just makes it into something" }, { "startMs": 910470, "text": "and it it just makes it into something" }, { "startMs": 910480, "text": "and it it just makes it into something that's a lot more autonomous and uh uh" }, { "startMs": 913030, "text": "that's a lot more autonomous and uh uh" }, { "startMs": 913040, "text": "that's a lot more autonomous and uh uh uh just a lot a lot better overall. And" }, { "startMs": 916470, "text": "uh just a lot a lot better overall. And" }, { "startMs": 916480, "text": "uh just a lot a lot better overall. And that is it. Thank you all so much for" }, { "startMs": 918069, "text": "that is it. Thank you all so much for" }, { "startMs": 918079, "text": "that is it. Thank you all so much for coming. Thank you." } ]